This is an automated email from the ASF dual-hosted git repository. lahirujayathilake pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata.git
commit 7716162705569bf32aa8703e0ed1b871419b7392 Author: yasith <[email protected]> AuthorDate: Sun Mar 2 04:11:24 2025 -0600 rename platform x86_64 -> amd64. fix issues in cyberfaces dockerfile. update notebooks. move environment files. --- modules/agent-framework/airavata-agent/Dockerfile | 2 +- modules/agent-framework/airavata-agent/Makefile | 2 +- .../deployments/agent-cerebrum/Makefile | 2 +- .../deployments/agent-cyberfaces/Dockerfile | 63 +- .../deployments/agent-cyberfaces/Makefile | 10 +- .../cyberfaces => agent-cyberfaces}/envs/aiml.yml | 1 - .../cyberfaces => agent-cyberfaces}/envs/fair.yml | 1 - .../envs/justiceindata.yml | 1 - .../cyberfaces => agent-cyberfaces}/envs/rs.yml | 1 - .../deployments/agent-i-guide/Makefile | 2 +- .../agent-framework/deployments/agent-md/Makefile | 2 +- .../deployments/agent-sample/Makefile | 2 +- .../deployments/jupyterhub/Dockerfile-jl-base | 2 +- .../deployments/jupyterhub/Dockerfile-jl-iguide | 2 +- .../cyberfaces/Solution_DA3_DEM_Access_v4.ipynb | 293 ++----- .../Solution_DP6_DEM_Processing_v2.ipynb | 839 ++------------------- 16 files changed, 134 insertions(+), 1091 deletions(-) diff --git a/modules/agent-framework/airavata-agent/Dockerfile b/modules/agent-framework/airavata-agent/Dockerfile index 5b6f8bc88d..116a1d314e 100644 --- a/modules/agent-framework/airavata-agent/Dockerfile +++ b/modules/agent-framework/airavata-agent/Dockerfile @@ -3,7 +3,7 @@ FROM mambaorg/micromamba:latest USER root RUN micromamba install -y -n base -c conda-forge \ - python=3.11 \ + "python<3.12" \ pip \ git \ flask \ diff --git a/modules/agent-framework/airavata-agent/Makefile b/modules/agent-framework/airavata-agent/Makefile index 5d82463f7b..ea30dfffff 100644 --- a/modules/agent-framework/airavata-agent/Makefile +++ b/modules/agent-framework/airavata-agent/Makefile @@ -4,7 +4,7 @@ AGENT_ID=test build-container: GOOS=linux GOARCH=amd64 go build -o airavata-agent && \ - docker build --platform linux/x86_64 -t $(IMAGE_NAME) . && \ + docker build --platform linux/amd64 -t $(IMAGE_NAME) . && \ docker push $(IMAGE_NAME) run-container: diff --git a/modules/agent-framework/deployments/agent-cerebrum/Makefile b/modules/agent-framework/deployments/agent-cerebrum/Makefile index cdb19da852..d971ef1b37 100644 --- a/modules/agent-framework/deployments/agent-cerebrum/Makefile +++ b/modules/agent-framework/deployments/agent-cerebrum/Makefile @@ -3,7 +3,7 @@ AGENT_SVC_URL=api.gateway.cybershuttle.org:19900 AGENT_ID=test build-container: - docker build --platform linux/x86_64 -t $(IMAGE_NAME) . && \ + docker build --platform linux/amd64 -t $(IMAGE_NAME) . && \ docker push $(IMAGE_NAME) run-container: diff --git a/modules/agent-framework/deployments/agent-cyberfaces/Dockerfile b/modules/agent-framework/deployments/agent-cyberfaces/Dockerfile index c8f169c57a..e451137da1 100644 --- a/modules/agent-framework/deployments/agent-cyberfaces/Dockerfile +++ b/modules/agent-framework/deployments/agent-cyberfaces/Dockerfile @@ -3,25 +3,36 @@ FROM cybershuttle/remote-agent-base:latest USER root RUN micromamba install -y -n base -c conda-forge \ + python \ + pip \ + ipykernel \ + numba==0.56.3 \ gdal \ - pandas \ - geopandas \ - pygeos \ - matplotlib \ - numpy \ - libpysal \ - esda \ - contextily \ + fiona \ + rasterio \ + ipyleaflet \ + scipy \ + bsddb3 \ + qgis \ + scikit-learn \ + && micromamba clean --all -y + +RUN micromamba run -n base pip install --no-cache-dir \ wget \ jupyter_contrib_nbextensions \ utm \ - matplotlib-scalebar \ + matplotlib \ + matplotlib_scalebar \ + geopandas \ + folium \ earthpy \ patool \ cdsapi \ + geemap \ gdown \ geojson \ pyarrow \ + xarray \ pooch \ zarr \ cython \ @@ -33,34 +44,12 @@ RUN micromamba install -y -n base -c conda-forge \ cmaps \ cartopy \ regionmask \ - cdo \ - progressbar2 \ + cdo-api-py \ + pynhd \ + progressbar \ + hydrofunctions \ googledrivedownloader \ + cartopy \ s3fs \ pywbt \ - cfgrib \ - tensorflow \ - seaborn \ - earthengine-api \ - folium \ - scikit-learn \ - netCDF4 \ - xarray \ - statsmodels \ - pmdarima \ - pygridmet \ - hydrofunctions \ - pynhd \ - missingno \ - geemap \ - numba \ - scipy \ - rasterio \ - ipyleaflet \ - bsddb3 \ - && micromamba clean --all -y - -RUN micromamba run -n base pip install --no-cache-dir --force-reinstall \ - cdo-api-py \ - "typing_extensions>=4.12.0" \ - "numpy<2.0.0" + cfgrib diff --git a/modules/agent-framework/deployments/agent-cyberfaces/Makefile b/modules/agent-framework/deployments/agent-cyberfaces/Makefile index e4c29b23f9..0022bd6c75 100644 --- a/modules/agent-framework/deployments/agent-cyberfaces/Makefile +++ b/modules/agent-framework/deployments/agent-cyberfaces/Makefile @@ -1,19 +1,17 @@ IMAGE_NAME=cybershuttle/remote-agent-cyberfaces AGENT_SVC_URL=api.gateway.cybershuttle.org:19900 AGENT_ID=test [email protected] -SIF_NAME=agent-framework/container/remote-agent-cyberfaces.sif build-container: - docker build --platform linux/x86_64 -t $(IMAGE_NAME) . && \ + docker build --platform linux/amd64 -t $(IMAGE_NAME) . && \ docker push $(IMAGE_NAME) run-container: docker run -it $(IMAGE_NAME) /opt/airavata-agent $(AGENT_SVC_URL) $(AGENT_ID) -deploy-container: - ssh $(SSH_IDENTIFIER) srun -p shared --mem 4G -t 30 singularity pull \ +deploy-anvil: + ssh [email protected] srun -p shared --mem 4G -t 30 singularity pull \ --disable-cache \ --force \ - $(SIF_NAME) \ + agent-framework/container/remote-agent-cyberfaces.sif \ docker://$(IMAGE_NAME) diff --git a/modules/agent-framework/deployments/jupyterhub/data/cyberfaces/envs/aiml.yml b/modules/agent-framework/deployments/agent-cyberfaces/envs/aiml.yml similarity index 96% rename from modules/agent-framework/deployments/jupyterhub/data/cyberfaces/envs/aiml.yml rename to modules/agent-framework/deployments/agent-cyberfaces/envs/aiml.yml index 5fc9eff641..c398ba58d7 100644 --- a/modules/agent-framework/deployments/jupyterhub/data/cyberfaces/envs/aiml.yml +++ b/modules/agent-framework/deployments/agent-cyberfaces/envs/aiml.yml @@ -7,7 +7,6 @@ dependencies: - numba==0.56.3 - scipy - pip: - - airavata-jupyter-magic - wget - jupyter_contrib_nbextensions - tensorflow diff --git a/modules/agent-framework/deployments/jupyterhub/data/cyberfaces/envs/fair.yml b/modules/agent-framework/deployments/agent-cyberfaces/envs/fair.yml similarity index 97% rename from modules/agent-framework/deployments/jupyterhub/data/cyberfaces/envs/fair.yml rename to modules/agent-framework/deployments/agent-cyberfaces/envs/fair.yml index a051df05d1..fd18e88ec4 100644 --- a/modules/agent-framework/deployments/jupyterhub/data/cyberfaces/envs/fair.yml +++ b/modules/agent-framework/deployments/agent-cyberfaces/envs/fair.yml @@ -19,7 +19,6 @@ dependencies: # - mplleaflet - pip: - - airavata-jupyter-magic - wget - jupyter_contrib_nbextensions - utm diff --git a/modules/agent-framework/deployments/jupyterhub/data/cyberfaces/envs/justiceindata.yml b/modules/agent-framework/deployments/agent-cyberfaces/envs/justiceindata.yml similarity index 66% rename from modules/agent-framework/deployments/jupyterhub/data/cyberfaces/envs/justiceindata.yml rename to modules/agent-framework/deployments/agent-cyberfaces/envs/justiceindata.yml index 9a04cd64b7..5155bb0357 100644 --- a/modules/agent-framework/deployments/jupyterhub/data/cyberfaces/envs/justiceindata.yml +++ b/modules/agent-framework/deployments/agent-cyberfaces/envs/justiceindata.yml @@ -6,5 +6,4 @@ dependencies: - ipykernel - netcdf4 - pip: - - airavata-jupyter-magic - dataretrieval \ No newline at end of file diff --git a/modules/agent-framework/deployments/jupyterhub/data/cyberfaces/envs/rs.yml b/modules/agent-framework/deployments/agent-cyberfaces/envs/rs.yml similarity index 87% rename from modules/agent-framework/deployments/jupyterhub/data/cyberfaces/envs/rs.yml rename to modules/agent-framework/deployments/agent-cyberfaces/envs/rs.yml index 0c634d7591..1d5ec5a3d2 100644 --- a/modules/agent-framework/deployments/jupyterhub/data/cyberfaces/envs/rs.yml +++ b/modules/agent-framework/deployments/agent-cyberfaces/envs/rs.yml @@ -9,7 +9,6 @@ dependencies: - libnetcdf - rasterio - pip: - - airavata-jupyter-magic - pyhdf - matplotlib - boto3 diff --git a/modules/agent-framework/deployments/agent-i-guide/Makefile b/modules/agent-framework/deployments/agent-i-guide/Makefile index 0d2c640ba4..c5953fc070 100644 --- a/modules/agent-framework/deployments/agent-i-guide/Makefile +++ b/modules/agent-framework/deployments/agent-i-guide/Makefile @@ -3,7 +3,7 @@ AGENT_SVC_URL=api.gateway.cybershuttle.org:19900 AGENT_ID=test build-container: - docker build --platform linux/x86_64 -t $(IMAGE_NAME) . && \ + docker build --platform linux/amd64 -t $(IMAGE_NAME) . && \ docker push $(IMAGE_NAME) run-container: diff --git a/modules/agent-framework/deployments/agent-md/Makefile b/modules/agent-framework/deployments/agent-md/Makefile index d1d6c1ec89..b3802cfb1d 100644 --- a/modules/agent-framework/deployments/agent-md/Makefile +++ b/modules/agent-framework/deployments/agent-md/Makefile @@ -3,7 +3,7 @@ AGENT_SVC_URL=api.gateway.cybershuttle.org:19900 AGENT_ID=test build-container: - docker build --platform linux/x86_64 -t $(IMAGE_NAME) . && \ + docker build --platform linux/amd64 -t $(IMAGE_NAME) . && \ docker push $(IMAGE_NAME) run-container: diff --git a/modules/agent-framework/deployments/agent-sample/Makefile b/modules/agent-framework/deployments/agent-sample/Makefile index cfd386a638..0d1dafcd2e 100644 --- a/modules/agent-framework/deployments/agent-sample/Makefile +++ b/modules/agent-framework/deployments/agent-sample/Makefile @@ -3,7 +3,7 @@ AGENT_SVC_URL=api.gateway.cybershuttle.org:19900 AGENT_ID=test build-container: - docker build --platform linux/x86_64 -t $(IMAGE_NAME) . && \ + docker build --platform linux/amd64 -t $(IMAGE_NAME) . && \ docker push $(IMAGE_NAME) run-container: diff --git a/modules/agent-framework/deployments/jupyterhub/Dockerfile-jl-base b/modules/agent-framework/deployments/jupyterhub/Dockerfile-jl-base index e115370edc..e7abb675ae 100644 --- a/modules/agent-framework/deployments/jupyterhub/Dockerfile-jl-base +++ b/modules/agent-framework/deployments/jupyterhub/Dockerfile-jl-base @@ -24,5 +24,5 @@ CMD micromamba run -n base jupyter lab \ --port=8888 \ --no-browser \ --allow-root \ - --NotebookApp.exec_lines='import airavata_magics' \ + --NotebookApp.exec_lines='import airavata_jupyter_magic' \ --NotebookApp.token='' diff --git a/modules/agent-framework/deployments/jupyterhub/Dockerfile-jl-iguide b/modules/agent-framework/deployments/jupyterhub/Dockerfile-jl-iguide index a49e2f78b4..c757409eab 100644 --- a/modules/agent-framework/deployments/jupyterhub/Dockerfile-jl-iguide +++ b/modules/agent-framework/deployments/jupyterhub/Dockerfile-jl-iguide @@ -26,5 +26,5 @@ CMD micromamba run -n base jupyter lab \ --port=8888 \ --no-browser \ --allow-root \ - --NotebookApp.exec_lines='import airavata_magics' \ + --NotebookApp.exec_lines='import airavata_jupyter_magic' \ --NotebookApp.token='' diff --git a/modules/agent-framework/deployments/jupyterhub/data/cyberfaces/Solution_DA3_DEM_Access_v4.ipynb b/modules/agent-framework/deployments/jupyterhub/data/cyberfaces/Solution_DA3_DEM_Access_v4.ipynb index 39cb8e96f3..b6cc9467a7 100644 --- a/modules/agent-framework/deployments/jupyterhub/data/cyberfaces/Solution_DA3_DEM_Access_v4.ipynb +++ b/modules/agent-framework/deployments/jupyterhub/data/cyberfaces/Solution_DA3_DEM_Access_v4.ipynb @@ -72,8 +72,8 @@ " %copy_data <r1:file1> <r2:file2> -- Copy <file1> in <r1> to <file2> in <r2>.\n", "\n", "\n", - "User code: YGEX-OJDG\n", - "Please authenticate by visiting: https://auth.cybershuttle.org/realms/default/device?user_code=YGEX-OJDG\n";, + "User code: YZJU-HUOU\n", + "Please authenticate by visiting: https://auth.cybershuttle.org/realms/default/device?user_code=YZJU-HUOU\n";, "Authorization pending, retrying...\n", "Received access token\n", "Requested runtime=test_cpu. state=14\n", @@ -99,7 +99,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "Waiting for runtime=test_cpu to be ready...\n", "Runtime=test_cpu is ready!\n" ] } @@ -210,99 +209,9 @@ }, { "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>TypeError: 'NoneType' object is not iterable</strong>\n", - " ---------------------------------------------------------------------------\n", - "TypeError Traceback (most recent call last)\n", - "Cell In[3], line 3\n", - " 1 ## Get the watershed using USGS station number using pynhd module\n", - " 2 ## WRITE THE CODE BELOW\n", - "----> 3 watershed = NLDI().get_basins(site_id)\n", - " 5 ## Plot the watershed\n", - " 6 ## DD indicates latitude/ longitude degrees is followed by immediately followed by decimal\n", - " 7 ax = watershed.plot(facecolor=\"b\", \n", - " 8 edgecolor=\"k\", \n", - " 9 figsize=(8, 8))\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/pynhd/pynhd.py:896, in NLDI.__init__(self)\n", - " 892 resp = cast(\"list[dict[str, Any]]\", resp)\n", - " 893 char_types = (\n", - " 894 ogc.traverse_json(r, [\"characteristicMetadata\", \"characteristic\"]) for r in resp\n", - " 895 )\n", - "--> 896 self.valid_characteristics = pd.concat(\n", - " 897 (pd.DataFrame(c) for c in char_types), ignore_index=True\n", - " 898 )\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/pandas/core/reshape/concat.py:382, in concat(objs, axis, join, ignore_index, keys, levels, names, verify_integrity, sort, copy)\n", - " 379 elif copy and using_copy_on_write():\n", - " 380 copy = False\n", - "--> 382 op = _Concatenator(\n", - " 383 objs,\n", - " 384 axis=axis,\n", - " 385 ignore_index=ignore_index,\n", - " 386 join=join,\n", - " 387 keys=keys,\n", - " 388 levels=levels,\n", - " 389 names=names,\n", - " 390 verify_integrity=verify_integrity,\n", - " 391 copy=copy,\n", - " 392 sort=sort,\n", - " 393 )\n", - " 395 return op.get_result()\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/pandas/core/reshape/concat.py:445, in _Concatenator.__init__(self, objs, axis, join, keys, levels, names, ignore_index, verify_integrity, copy, sort)\n", - " 442 self.verify_integrity = verify_integrity\n", - " 443 self.copy = copy\n", - "--> 445 objs, keys = self._clean_keys_and_objs(objs, keys)\n", - " 447 # figure out what our result ndim is going to be\n", - " 448 ndims = self._get_ndims(objs)\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/pandas/core/reshape/concat.py:504, in _Concatenator._clean_keys_and_objs(self, objs, keys)\n", - " 502 objs_list = [objs[k] for k in keys]\n", - " 503 else:\n", - "--> 504 objs_list = list(objs)\n", - " 506 if len(objs_list) == 0:\n", - " 507 raise ValueError(\"No objects to concatenate\")\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/pynhd/pynhd.py:897, in <genexpr>(.0)\n", - " 892 resp = cast(\"list[dict[str, Any]]\", resp)\n", - " 893 char_types = (\n", - " 894 ogc.traverse_json(r, [\"characteristicMetadata\", \"characteristic\"]) for r in resp\n", - " 895 )\n", - " 896 self.valid_characteristics = pd.concat(\n", - "--> 897 (pd.DataFrame(c) for c in char_types), ignore_index=True\n", - " 898 )\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/pynhd/pynhd.py:894, in <genexpr>(.0)\n", - " 891 resp = ar.retrieve_json([r[\"characteristics\"] for r in resp[0]])\n", - " 892 resp = cast(\"list[dict[str, Any]]\", resp)\n", - " 893 char_types = (\n", - "--> 894 ogc.traverse_json(r, [\"characteristicMetadata\", \"characteristic\"]) for r in resp\n", - " 895 )\n", - " 896 self.valid_characteristics = pd.concat(\n", - " 897 (pd.DataFrame(c) for c in char_types), ignore_index=True\n", - " 898 )\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/pygeoogc/utils.py:515, in traverse_json(json_data, ipath)\n", - " 512 if isinstance(json_data, dict):\n", - " 513 return extract(json_data, ipath, 0, [])\n", - "--> 515 outer_arr = [extract(item, ipath, 0, []) for item in json_data]\n", - " 516 return outer_arr\n", - "\n", - "TypeError: 'NoneType' object is not iterable\n", - "</pre></div>" - ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAArwAAAG9CAYAAAD3DqV7AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAArKtJREFUeJzs3Xd8U9Ubx/HPTdJBgZY9yt6FMoWy9x6CIEMEWSo4kCEoU2QqiAgoyB4KoiAyRJS9t8hWdgVktUCBlraUNrnn90fa/Cgt0ELaW9rn/XpdTW5OTr4pafv03nPP0ZRSCiGEEEIIIVIpk9EBhBBCCCGESEpS8AohhBBCiFRNCl4hhBBCCJGqScErhBBCCCFSNSl4hRBCCCFEqiYFrxBCCCGESNWk4BVCCCGEEKmaFLxCCCGEECJVk4JXCCGEEEKkalLwCpEMNE3jgw8+MDrGU61fv57y5cvj7u6Opmnc [...] "text/plain": [ - "<IPython.core.display.HTML object>" + "<IPython.core.display.Image object>" ] }, "metadata": {}, @@ -356,39 +265,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "Runtime=test_cpu is ready!\n" + "Runtime=test_cpu is ready!\n", + "Left Bounding Longtitude is -87.208° or 87.208° W\n", + "Right Bounding Longtitude is -84.407° or 84.407° W\n", + "Bottom Bounding Latitude is 40.121° or 40.121° N\n", + "Top Bounding Latitude is 41.354° or 41.354° N\n" ] - }, - { - "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>NameError: name 'watershed' is not defined</strong>\n", - " ---------------------------------------------------------------------------\n", - "NameError Traceback (most recent call last)\n", - "Cell In[4], line 2\n", - " 1 ## Get the min and max of latitude and longitude (or easting and northing)\n", - "----> 2 extents_basin=watershed.total_bounds\n", - " 4 ## N or S and W or E may become a problem\n", - " 5 print(f'Left Bounding Longtitude is {extents_basin[0]:.3f}\\u00b0 or {abs(extents_basin[0]):.3f}\\u00b0 W')\n", - "\n", - "NameError: name 'watershed' is not defined\n", - "</pre></div>" - ], - "text/plain": [ - "<IPython.core.display.HTML object>" - ] - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ @@ -415,43 +297,6 @@ "text": [ "Runtime=test_cpu is ready!\n" ] - }, - { - "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>NameError: name 'extents_basin' is not defined</strong>\n", - " ---------------------------------------------------------------------------\n", - "NameError Traceback (most recent call last)\n", - "Cell In[5], line 4\n", - " 1 ## DEMs are numbered using integer\n", - " 2 ## Calculate largest integer that equals or not greater than left and bottom bounds\n", - " 3 ## WRITE THE CODE BELOW\n", - "----> 4 extent_left=abs(math.floor(extents_basin[0]))\n", - " 5 extent_right=abs(math.floor(extents_basin[2]))\n", - " 6 ## You may be tempted to calculate the ceil of right extent\n", - " 7 ## But, number scheme is such that 84W indicates data from -84 to -83 deg W\n", - " 8 \n", - " 9 ## Calculate smallest integer that equals or not less than right and upper bounds\n", - " 10 ## WRITE THE CODE BELOW\n", - "\n", - "NameError: name 'extents_basin' is not defined\n", - "</pre></div>" - ], - "text/plain": [ - "<IPython.core.display.HTML object>" - ] - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ @@ -493,41 +338,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "Runtime=test_cpu is ready!\n" + "Runtime=test_cpu is ready!\n", + "The required lon and lat pairs are: \n", + " [(85, 41), (86, 41), (86, 42), (87, 41), (87, 42), (88, 41)]\n", + "\n", + "Number of tiles required to cover the entire region: 8\n", + "Left: 88, Right: 85, Bottom: 41, Top: 42\n", + "\n", + "Number of tiles within watershed boundary: 6\n" ] - }, - { - "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>NameError: name 'extent_right' is not defined</strong>\n", - " ---------------------------------------------------------------------------\n", - "NameError Traceback (most recent call last)\n", - "Cell In[6], line 5\n", - " 2 overlap_lonlat=[]\n", - " 4 ## Create a for loop to create a rectangular boundary and see if overlaps with watershed\n", - "----> 5 for lon in (range(extent_right,extent_left+1,1)):\n", - " 6 for lat in (range(extent_bottom,extent_top+1,1)):\n", - " 7 ## Defining in anticlockwise direction\n", - " 8 corner_left_bottom=(-lon,lat-1)\n", - "\n", - "NameError: name 'extent_right' is not defined\n", - "</pre></div>" - ], - "text/plain": [ - "<IPython.core.display.HTML object>" - ] - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ @@ -629,8 +448,36 @@ "name": "stdout", "output_type": "stream", "text": [ - "Runtime=test_cpu is ready!\n" + "Runtime=test_cpu is ready!\n", + "Latitude: 41 N ; , Longitude: 85 W\n", + "Beginning file 1 download with urllib2 out of 6...\n", + "Data downloaded from : \n", + "https://prd-tnm.s3.amazonaws.com/StagedProducts/Elevation/1/TIFF/current/n41w085/USGS_1_n41w085.tif\n"; ] + }, + { + "data": { + "text/html": [ + "\n", + " <div style=\"\n", + " color: #a71d5d;\n", + " background-color: #fdd;\n", + " border: 1px solid #a71d5d;\n", + " padding: 5px;\n", + " border-radius: 5px;\n", + " font-family: Consolas, 'Courier New', monospace;\n", + " white-space: pre-wrap;\n", + " \">\n", + " 0% | |\n", + " </div>\n", + " " + ], + "text/plain": [ + "<IPython.core.display.HTML object>" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ @@ -684,40 +531,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "Runtime=test_cpu is ready!\n" + "Runtime=test_cpu is ready!\n", + "Text(45.222222222222214, 0.5, 'Longitude (DD)')\n" ] }, { "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>NameError: name 'watershed' is not defined</strong>\n", - " ---------------------------------------------------------------------------\n", - "NameError Traceback (most recent call last)\n", - "Cell In[9], line 12\n", - " 8 rasterio.plot.show(raster,\n", - " 9 ax=ax,\n", - " 10 cmap='viridis')\n", - " 11 #print(f'lat: {lat},lon: {lon},file:{local_fileloc_filename}')\n", - "---> 12 watershed.plot(ax=ax, \n", - " 13 facecolor='none', \n", - " 14 edgecolor='red')\n", - " 15 plt.title(\"Unmerged Raster DEMs\")\n", - " 16 plt.xlabel(\"Longitude (DD)\")\n", - "\n", - "NameError: name 'watershed' is not defined\n", - "</pre></div>" - ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtkAAAHvCAYAAACBhsOzAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzsvXu0NVlZ3vt755xVtS778t36SjcXGxSabjXAEI2BxjQgodXIAT0tMAQhgiOgGOTo8IRDNKKgHo16GEhIjHoEQw5eYi5DEzpCTIwhRLwAEQMKiLRNX77Lvqy16jLne/54Z9Va+/u+BsRuuvlSzxi79961alXNmrV2f8986nmfV1RVGTFixIgRI0aMGDFixH0G90APYMSIESNGjBgxYsSISw0jyR4xYsSIESNGjBgx4j7GSLJHjBgxYsSIESNGjLiPMZLsESNGjBgxYsSIESPuY4wke8SIESNGjBgx [...] "text/plain": [ - "<IPython.core.display.HTML object>" + "<IPython.core.display.Image object>" ] }, "metadata": {}, @@ -746,18 +568,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Terminated runtime=test_cpu. state={'experimentId': 'CS_Agent_cfd825be-8531-4103-a9c9-77dace080fcd', 'terminated': True}\n", - "Switched to runtime=local.\n" - ] - } - ], + "outputs": [], "source": [ "%stop_runtime test_cpu\n", "%switch_runtime local" @@ -766,7 +579,7 @@ ], "metadata": { "kernelspec": { - "display_name": "airavata_examples", + "display_name": "airavata", "language": "python", "name": "python3" }, diff --git a/modules/agent-framework/deployments/jupyterhub/data/cyberfaces/Solution_DP6_DEM_Processing_v2.ipynb b/modules/agent-framework/deployments/jupyterhub/data/cyberfaces/Solution_DP6_DEM_Processing_v2.ipynb index 676b62a712..b5f3b4d147 100644 --- a/modules/agent-framework/deployments/jupyterhub/data/cyberfaces/Solution_DP6_DEM_Processing_v2.ipynb +++ b/modules/agent-framework/deployments/jupyterhub/data/cyberfaces/Solution_DP6_DEM_Processing_v2.ipynb @@ -73,8 +73,9 @@ " %copy_data <r1:file1> <r2:file2> -- Copy <file1> in <r1> to <file2> in <r2>.\n", "\n", "\n", - "User code: CRRQ-PFFQ\n", - "Please authenticate by visiting: https://auth.cybershuttle.org/realms/default/device?user_code=CRRQ-PFFQ\n";, + "User code: IZXC-STVJ\n", + "Please authenticate by visiting: https://auth.cybershuttle.org/realms/default/device?user_code=IZXC-STVJ\n";, + "Authorization pending, retrying...\n", "Authorization pending, retrying...\n", "Received access token\n", "Requested runtime=test_cpu. state=14\n", @@ -102,6 +103,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "Waiting for runtime=test_cpu to be ready...\n", "Runtime=test_cpu is ready!\n" ] } @@ -144,9 +146,9 @@ "output_type": "stream", "text": [ "Runtime=test_cpu is ready!\n", - "1.3.8\n", - "0.14.4\n", - "3.6.1\n" + "1.4.3\n", + "1.0.1\n", + "3.7.1\n" ] } ], @@ -177,98 +179,21 @@ "name": "stdout", "output_type": "stream", "text": [ - "Runtime=test_cpu is ready!\n" + "Runtime=test_cpu is ready!\n", + "The required lon and lat pairs are: \n", + " [(85, 41), (86, 41), (86, 42), (87, 41), (87, 42), (88, 41)]\n", + "\n", + "Number of tiles required to cover the entire region: 8\n", + "Left: 88, Right: 85, Bottom: 41, Top: 42\n", + "\n", + "Number of tiles within watershed boundary: 6\n" ] }, { "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>DriverError: ./input_03335500/shape_03335500.shp: No such file or directory</strong>\n", - " ---------------------------------------------------------------------------\n", - "CPLE_OpenFailedError Traceback (most recent call last)\n", - "File fiona/ogrext.pyx:136, in fiona.ogrext.gdal_open_vector()\n", - "\n", - "File fiona/_err.pyx:291, in fiona._err.exc_wrap_pointer()\n", - "\n", - "CPLE_OpenFailedError: ./input_03335500/shape_03335500.shp: No such file or directory\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "DriverError Traceback (most recent call last)\n", - "Cell In[3], line 13\n", - " 11 dem_files_store= folder_input\n", - " 12 ##Read the shapefile using geopandas\n", - "---> 13 watershed =geopandas.read_file(f'{folder_input}/shape_{site_id}.shp')\n", - " 14 ## Get the extents and number of tiles downloaded\n", - " 15 extents_basin=watershed.total_bounds\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/geopandas/io/file.py:289, in _read_file(filename, bbox, mask, rows, engine, **kwargs)\n", - " 286 else:\n", - " 287 path_or_bytes = filename\n", - "--> 289 return _read_file_fiona(\n", - " 290 path_or_bytes, from_bytes, bbox=bbox, mask=mask, rows=rows, **kwargs\n", - " 291 )\n", - " 293 else:\n", - " 294 raise ValueError(f\"unknown engine '{engine}'\")\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/geopandas/io/file.py:315, in _read_file_fiona(path_or_bytes, from_bytes, bbox, mask, rows, where, **kwargs)\n", - " 312 reader = fiona.open\n", - " 314 with fiona_env():\n", - "--> 315 with reader(path_or_bytes, **kwargs) as features:\n", - " 316 crs = features.crs_wkt\n", - " 317 # attempt to get EPSG code\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/fiona/env.py:457, in ensure_env_with_credentials.<locals>.wrapper(*args, **kwds)\n", - " 454 session = DummySession()\n", - " 456 with env_ctor(session=session):\n", - "--> 457 return f(*args, **kwds)\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/fiona/__init__.py:292, in open(fp, mode, driver, schema, crs, encoding, layer, vfs, enabled_drivers, crs_wkt, allow_unsupported_drivers, **kwargs)\n", - " 289 path = parse_path(fp)\n", - " 291 if mode in (\"a\", \"r\"):\n", - "--> 292 colxn = Collection(\n", - " 293 path,\n", - " 294 mode,\n", - " 295 driver=driver,\n", - " 296 encoding=encoding,\n", - " 297 layer=layer,\n", - " 298 enabled_drivers=enabled_drivers,\n", - " 299 allow_unsupported_drivers=allow_unsupported_drivers,\n", - " 300 **kwargs\n", - " 301 )\n", - " 302 elif mode == \"w\":\n", - " 303 colxn = Collection(\n", - " 304 path,\n", - " 305 mode,\n", - " (...)\n", - " 314 **kwargs\n", - " 315 )\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/fiona/collection.py:243, in Collection.__init__(self, path, mode, driver, schema, crs, encoding, layer, vsi, archive, enabled_drivers, crs_wkt, ignore_fields, ignore_geometry, include_fields, wkt_version, allow_unsupported_drivers, **kwargs)\n", - " 241 if self.mode == \"r\":\n", - " 242 self.session = Session()\n", - "--> 243 self.session.start(self, **kwargs)\n", - " 244 elif self.mode in (\"a\", \"w\"):\n", - " 245 self.session = WritingSession()\n", - "\n", - "File fiona/ogrext.pyx:588, in fiona.ogrext.Session.start()\n", - "\n", - "File fiona/ogrext.pyx:143, in fiona.ogrext.gdal_open_vector()\n", - "\n", - "DriverError: ./input_03335500/shape_03335500.shp: No such file or directory\n", - "</pre></div>" - ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtkAAAIDCAYAAADR6ALXAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzsvXm8JVV57v9dQ9UezthNdwNNM4kaBDQKzkbBCyoCGr2iweHGgRtNIg6JXvNJJEajUaKJmlx/imYyUdRc5yReNaByzVUvojhh1BhFRZChobvPsPeuYa3398e7qvY+3Y2goiDW8/kc+uzatatWrdpNP+up531eIyJChw4dOnTo0KFDhw4dbjXY23oAHTp06NChQ4cOHTrc0dCR7A4dOnTo0KFDhw4dbmV0JLtDhw4dOnTo0KFDh1sZHcnu0KFDhw4dOnTo0OFWRkeyO3To0KFDhw4dOnS4ldGR7A4d [...] "text/plain": [ - "<IPython.core.display.HTML object>" + "<IPython.core.display.Image object>" ] }, "metadata": {}, @@ -447,40 +372,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Runtime=test_cpu is ready!\n" + "Runtime=test_cpu is ready!\n", + "Merging completed for 6 DEM raster files\n" ] - }, - { - "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>NameError: name 'num_tiles_download' is not defined</strong>\n", - " ---------------------------------------------------------------------------\n", - "NameError Traceback (most recent call last)\n", - "Cell In[5], line 48\n", - " 42 del mosaic_new_raster\n", - " 47 ## WRITE THE CODE BELOW \n", - "---> 48 if num_tiles_download==1:\n", - " 49 merged_raster_filename=f'{dem_files_store}/USGS_{resolution}_{usgs_filename}.tif'\n", - " 50 print(\"No need to merge as only one tiff file\")\n", - "\n", - "NameError: name 'num_tiles_download' is not defined\n", - "</pre></div>" - ], - "text/plain": [ - "<IPython.core.display.HTML object>" - ] - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ @@ -575,7 +469,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "tags": [] }, @@ -586,36 +480,6 @@ "text": [ "Runtime=test_cpu is ready!\n" ] - }, - { - "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>NameError: name 'merged_raster_filename' is not defined</strong>\n", - " ---------------------------------------------------------------------------\n", - "NameError Traceback (most recent call last)\n", - "Cell In[7], line 1\n", - "----> 1 with rasterio.open(merged_raster_filename) as src:\n", - " 2 print(src)\n", - " 3 fig, ax = plt.subplots(figsize = (8,8))\n", - "\n", - "NameError: name 'merged_raster_filename' is not defined\n", - "</pre></div>" - ], - "text/plain": [ - "<IPython.core.display.HTML object>" - ] - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ @@ -657,48 +521,11 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Runtime=test_cpu is ready!\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>NameError: name 'watershed' is not defined</strong>\n", - " ---------------------------------------------------------------------------\n", - "NameError Traceback (most recent call last)\n", - "Cell In[8], line 2\n", - " 1 ## Find if basin has a projection\n", - "----> 2 print(watershed.crs)\n", - "\n", - "NameError: name 'watershed' is not defined\n", - "</pre></div>" - ], - "text/plain": [ - "<IPython.core.display.HTML object>" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "## Find if basin has a projection\n", "print(watershed.crs)" @@ -718,50 +545,11 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Runtime=test_cpu is ready!\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>NameError: name 'watershed' is not defined</strong>\n", - " ---------------------------------------------------------------------------\n", - "NameError Traceback (most recent call last)\n", - "Cell In[9], line 2\n", - " 1 ## Find the approximate centroid\n", - "----> 2 df=watershed\n", - " 3 df['geometry'].centroid\n", - " 4 ## Find the center point\n", - "\n", - "NameError: name 'watershed' is not defined\n", - "</pre></div>" - ], - "text/plain": [ - "<IPython.core.display.HTML object>" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "## Find the approximate centroid\n", "df=watershed\n", @@ -776,50 +564,11 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Runtime=test_cpu is ready!\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>NameError: name 'df' is not defined</strong>\n", - " ---------------------------------------------------------------------------\n", - "NameError Traceback (most recent call last)\n", - "Cell In[10], line 2\n", - " 1 ## Calculate N or S\n", - "----> 2 if 0 < df['lat'][0] < 84:\n", - " 3 lat_zone='N'\n", - " 4 elif -84 < df['lat'][0] < 0:\n", - "\n", - "NameError: name 'df' is not defined\n", - "</pre></div>" - ], - "text/plain": [ - "<IPython.core.display.HTML object>" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "## Calculate N or S\n", "if 0 < df['lat'][0] < 84:\n", @@ -869,49 +618,11 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Runtime=test_cpu is ready!\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>NameError: name 'watershed' is not defined</strong>\n", - " ---------------------------------------------------------------------------\n", - "NameError Traceback (most recent call last)\n", - "Cell In[11], line 2\n", - " 1 ## Alternative Method to estimate the CRS\n", - "----> 2 crs_string2=watershed.estimate_utm_crs(datum_name='WGS 84')\n", - " 3 crs_string2\n", - "\n", - "NameError: name 'watershed' is not defined\n", - "</pre></div>" - ], - "text/plain": [ - "<IPython.core.display.HTML object>" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "## Alternative Method to estimate the CRS\n", "crs_string2=watershed.estimate_utm_crs(datum_name='WGS 84')\n", @@ -931,111 +642,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Runtime=test_cpu is ready!\n", - "The new directory \u001b[1m'./input_03335500/results_03335500'\u001b[0m is not created as it already exists!\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>DriverError: ./input_03335500/shape_03335500.shp: No such file or directory</strong>\n", - " ---------------------------------------------------------------------------\n", - "CPLE_OpenFailedError Traceback (most recent call last)\n", - "File fiona/ogrext.pyx:136, in fiona.ogrext.gdal_open_vector()\n", - "\n", - "File fiona/_err.pyx:291, in fiona._err.exc_wrap_pointer()\n", - "\n", - "CPLE_OpenFailedError: ./input_03335500/shape_03335500.shp: No such file or directory\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "DriverError Traceback (most recent call last)\n", - "Cell In[12], line 6\n", - " 3 check_create_path_func(folder_results)\n", - " 5 shapefile_fileloc_filename=f'{folder_input}/shape_{site_id}.shp'\n", - "----> 6 watershed_file=geopandas.read_file(shapefile_fileloc_filename)\n", - " 7 watershed_proj=watershed_file.to_crs(crs_string)\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/geopandas/io/file.py:289, in _read_file(filename, bbox, mask, rows, engine, **kwargs)\n", - " 286 else:\n", - " 287 path_or_bytes = filename\n", - "--> 289 return _read_file_fiona(\n", - " 290 path_or_bytes, from_bytes, bbox=bbox, mask=mask, rows=rows, **kwargs\n", - " 291 )\n", - " 293 else:\n", - " 294 raise ValueError(f\"unknown engine '{engine}'\")\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/geopandas/io/file.py:315, in _read_file_fiona(path_or_bytes, from_bytes, bbox, mask, rows, where, **kwargs)\n", - " 312 reader = fiona.open\n", - " 314 with fiona_env():\n", - "--> 315 with reader(path_or_bytes, **kwargs) as features:\n", - " 316 crs = features.crs_wkt\n", - " 317 # attempt to get EPSG code\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/fiona/env.py:457, in ensure_env_with_credentials.<locals>.wrapper(*args, **kwds)\n", - " 454 session = DummySession()\n", - " 456 with env_ctor(session=session):\n", - "--> 457 return f(*args, **kwds)\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/fiona/__init__.py:292, in open(fp, mode, driver, schema, crs, encoding, layer, vfs, enabled_drivers, crs_wkt, allow_unsupported_drivers, **kwargs)\n", - " 289 path = parse_path(fp)\n", - " 291 if mode in (\"a\", \"r\"):\n", - "--> 292 colxn = Collection(\n", - " 293 path,\n", - " 294 mode,\n", - " 295 driver=driver,\n", - " 296 encoding=encoding,\n", - " 297 layer=layer,\n", - " 298 enabled_drivers=enabled_drivers,\n", - " 299 allow_unsupported_drivers=allow_unsupported_drivers,\n", - " 300 **kwargs\n", - " 301 )\n", - " 302 elif mode == \"w\":\n", - " 303 colxn = Collection(\n", - " 304 path,\n", - " 305 mode,\n", - " (...)\n", - " 314 **kwargs\n", - " 315 )\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/fiona/collection.py:243, in Collection.__init__(self, path, mode, driver, schema, crs, encoding, layer, vsi, archive, enabled_drivers, crs_wkt, ignore_fields, ignore_geometry, include_fields, wkt_version, allow_unsupported_drivers, **kwargs)\n", - " 241 if self.mode == \"r\":\n", - " 242 self.session = Session()\n", - "--> 243 self.session.start(self, **kwargs)\n", - " 244 elif self.mode in (\"a\", \"w\"):\n", - " 245 self.session = WritingSession()\n", - "\n", - "File fiona/ogrext.pyx:588, in fiona.ogrext.Session.start()\n", - "\n", - "File fiona/ogrext.pyx:143, in fiona.ogrext.gdal_open_vector()\n", - "\n", - "DriverError: ./input_03335500/shape_03335500.shp: No such file or directory\n", - "</pre></div>" - ], - "text/plain": [ - "<IPython.core.display.HTML object>" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "folder_results=f'{folder_input}/results_{site_id}'\n", "#folder_results=f'C:/temp/JibinJoseph/GDAL_operations/results_{site_id}'\n", @@ -1048,49 +657,11 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Runtime=test_cpu is ready!\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>NameError: name 'watershed_file' is not defined</strong>\n", - " ---------------------------------------------------------------------------\n", - "NameError Traceback (most recent call last)\n", - "Cell In[13], line 1\n", - "----> 1 print(watershed_file.crs)\n", - " 2 print(watershed_proj.crs)\n", - " 3 print(type(watershed_file))\n", - "\n", - "NameError: name 'watershed_file' is not defined\n", - "</pre></div>" - ], - "text/plain": [ - "<IPython.core.display.HTML object>" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "print(watershed_file.crs)\n", "print(watershed_proj.crs)\n", @@ -1100,50 +671,11 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Runtime=test_cpu is ready!\n", - "./input_03335500/results_03335500/proj_shp_03335500.shp\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>NameError: name 'watershed_proj' is not defined</strong>\n", - " ---------------------------------------------------------------------------\n", - "NameError Traceback (most recent call last)\n", - "Cell In[14], line 3\n", - " 1 proj_shapefile_filename=f'{folder_results}/proj_shp_{site_id}.shp'\n", - " 2 print(proj_shapefile_filename)\n", - "----> 3 watershed_proj.to_file(proj_shapefile_filename, driver='ESRI Shapefile',mode='w')\n", - "\n", - "NameError: name 'watershed_proj' is not defined\n", - "</pre></div>" - ], - "text/plain": [ - "<IPython.core.display.HTML object>" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "proj_shapefile_filename=f'{folder_results}/proj_shp_{site_id}.shp'\n", "print(proj_shapefile_filename)\n", @@ -1163,50 +695,11 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Runtime=test_cpu is ready!\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>NameError: name 'merged_raster_filename' is not defined</strong>\n", - " ---------------------------------------------------------------------------\n", - "NameError Traceback (most recent call last)\n", - "Cell In[15], line 35\n", - " 32 ## WRITE CODE BELOW\n", - " 34 reprojected_raster_filename=f'{folder_intermediate}/reprojected_{resolution}_{site_id}.tif'\n", - "---> 35 print(merged_raster_filename,\"\\n\",reprojected_raster_filename)\n", - " 36 reproject_raster_func(merged_raster_filename, reprojected_raster_filename, crs_string)\n", - "\n", - "NameError: name 'merged_raster_filename' is not defined\n", - "</pre></div>" - ], - "text/plain": [ - "<IPython.core.display.HTML object>" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "def reproject_raster_func(input_raster, output_raster, target_crs):\n", " # Load the input DEM\n", @@ -1248,79 +741,11 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Runtime=test_cpu is ready!\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>RasterioIOError: ./input_03335500/intermediate_03335500/reprojected_1_03335500.tif: No such file or directory</strong>\n", - " ---------------------------------------------------------------------------\n", - "CPLE_OpenFailedError Traceback (most recent call last)\n", - "File rasterio/_base.pyx:310, in rasterio._base.DatasetBase.__init__()\n", - "\n", - "File rasterio/_base.pyx:221, in rasterio._base.open_dataset()\n", - "\n", - "File rasterio/_err.pyx:221, in rasterio._err.exc_wrap_pointer()\n", - "\n", - "CPLE_OpenFailedError: ./input_03335500/intermediate_03335500/reprojected_1_03335500.tif: No such file or directory\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "RasterioIOError Traceback (most recent call last)\n", - "Cell In[16], line 2\n", - " 1 reprojected_raster_filename=f'{folder_intermediate}/reprojected_{resolution}_{site_id}.tif'\n", - "----> 2 with rasterio.open(reprojected_raster_filename) as src:\n", - " 3 fig, ax = plt.subplots(figsize = (8,8))\n", - " 4 mapped=rasterio.plot.show(src,\n", - " 5 ax=ax, \n", - " 6 cmap='viridis')\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/rasterio/env.py:451, in ensure_env_with_credentials.<locals>.wrapper(*args, **kwds)\n", - " 448 session = DummySession()\n", - " 450 with env_ctor(session=session):\n", - "--> 451 return f(*args, **kwds)\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/rasterio/__init__.py:304, in open(fp, mode, driver, width, height, count, crs, transform, dtype, nodata, sharing, **kwargs)\n", - " 301 path = _parse_path(raw_dataset_path)\n", - " 303 if mode == \"r\":\n", - "--> 304 dataset = DatasetReader(path, driver=driver, sharing=sharing, **kwargs)\n", - " 305 elif mode == \"r+\":\n", - " 306 dataset = get_writer_for_path(path, driver=driver)(\n", - " 307 path, mode, driver=driver, sharing=sharing, **kwargs\n", - " 308 )\n", - "\n", - "File rasterio/_base.pyx:312, in rasterio._base.DatasetBase.__init__()\n", - "\n", - "RasterioIOError: ./input_03335500/intermediate_03335500/reprojected_1_03335500.tif: No such file or directory\n", - "</pre></div>" - ], - "text/plain": [ - "<IPython.core.display.HTML object>" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "reprojected_raster_filename=f'{folder_intermediate}/reprojected_{resolution}_{site_id}.tif'\n", "with rasterio.open(reprojected_raster_filename) as src:\n", @@ -1362,122 +787,11 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Runtime=test_cpu is ready!\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>DriverError: ./input_03335500/results_03335500/proj_shp_03335500.shp: No such file or directory</strong>\n", - " ---------------------------------------------------------------------------\n", - "CPLE_OpenFailedError Traceback (most recent call last)\n", - "File fiona/ogrext.pyx:136, in fiona.ogrext.gdal_open_vector()\n", - "\n", - "File fiona/_err.pyx:291, in fiona._err.exc_wrap_pointer()\n", - "\n", - "CPLE_OpenFailedError: ./input_03335500/results_03335500/proj_shp_03335500.shp: No such file or directory\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "DriverError Traceback (most recent call last)\n", - "Cell In[17], line 30\n", - " 27 buffer_value_in_meters=1500\n", - " 29 clipped_raster_filename=f'{folder_results}/clipped_{resolution}_{site_id}.tif'\n", - "---> 30 clip_raster_with_shapefile_func(reprojected_raster_filename,\n", - " 31 clipped_raster_filename,\n", - " 32 proj_shapefile_filename,\n", - " 33 buffer_value_in_meters)\n", - "\n", - "Cell In[17], line 5, in clip_raster_with_shapefile_func(input_raster, output_raster, shapefile, buffer_value)\n", - " 3 def clip_raster_with_shapefile_func(input_raster, output_raster, shapefile,buffer_value):\n", - " 4 ## Open the shapefile using geopandas\n", - "----> 5 shapefile_gdf = geopandas.read_file(shapefile).buffer(buffer_value)\n", - " 6 ## Open the input raster\n", - " 7 with rasterio.open(input_raster) as src:\n", - " 8 ## Convert the shapefile geometry to the same CRS as the raster\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/geopandas/io/file.py:289, in _read_file(filename, bbox, mask, rows, engine, **kwargs)\n", - " 286 else:\n", - " 287 path_or_bytes = filename\n", - "--> 289 return _read_file_fiona(\n", - " 290 path_or_bytes, from_bytes, bbox=bbox, mask=mask, rows=rows, **kwargs\n", - " 291 )\n", - " 293 else:\n", - " 294 raise ValueError(f\"unknown engine '{engine}'\")\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/geopandas/io/file.py:315, in _read_file_fiona(path_or_bytes, from_bytes, bbox, mask, rows, where, **kwargs)\n", - " 312 reader = fiona.open\n", - " 314 with fiona_env():\n", - "--> 315 with reader(path_or_bytes, **kwargs) as features:\n", - " 316 crs = features.crs_wkt\n", - " 317 # attempt to get EPSG code\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/fiona/env.py:457, in ensure_env_with_credentials.<locals>.wrapper(*args, **kwds)\n", - " 454 session = DummySession()\n", - " 456 with env_ctor(session=session):\n", - "--> 457 return f(*args, **kwds)\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/fiona/__init__.py:292, in open(fp, mode, driver, schema, crs, encoding, layer, vfs, enabled_drivers, crs_wkt, allow_unsupported_drivers, **kwargs)\n", - " 289 path = parse_path(fp)\n", - " 291 if mode in (\"a\", \"r\"):\n", - "--> 292 colxn = Collection(\n", - " 293 path,\n", - " 294 mode,\n", - " 295 driver=driver,\n", - " 296 encoding=encoding,\n", - " 297 layer=layer,\n", - " 298 enabled_drivers=enabled_drivers,\n", - " 299 allow_unsupported_drivers=allow_unsupported_drivers,\n", - " 300 **kwargs\n", - " 301 )\n", - " 302 elif mode == \"w\":\n", - " 303 colxn = Collection(\n", - " 304 path,\n", - " 305 mode,\n", - " (...)\n", - " 314 **kwargs\n", - " 315 )\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/fiona/collection.py:243, in Collection.__init__(self, path, mode, driver, schema, crs, encoding, layer, vsi, archive, enabled_drivers, crs_wkt, ignore_fields, ignore_geometry, include_fields, wkt_version, allow_unsupported_drivers, **kwargs)\n", - " 241 if self.mode == \"r\":\n", - " 242 self.session = Session()\n", - "--> 243 self.session.start(self, **kwargs)\n", - " 244 elif self.mode in (\"a\", \"w\"):\n", - " 245 self.session = WritingSession()\n", - "\n", - "File fiona/ogrext.pyx:588, in fiona.ogrext.Session.start()\n", - "\n", - "File fiona/ogrext.pyx:143, in fiona.ogrext.gdal_open_vector()\n", - "\n", - "DriverError: ./input_03335500/results_03335500/proj_shp_03335500.shp: No such file or directory\n", - "</pre></div>" - ], - "text/plain": [ - "<IPython.core.display.HTML object>" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "proj_shapefile_filename=f'{folder_results}/proj_shp_{site_id}.shp'\n", "\n", @@ -1516,76 +830,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Runtime=test_cpu is ready!\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " <div style=\"\n", - " color: #a71d5d;\n", - " background-color: #fdd;\n", - " border: 1px solid #a71d5d;\n", - " padding: 5px;\n", - " border-radius: 5px;\n", - " font-family: Consolas, 'Courier New', monospace;\n", - " \">\n", - " <pre><strong>RasterioIOError: ./input_03335500/results_03335500/clipped_1_03335500.tif: No such file or directory</strong>\n", - " ---------------------------------------------------------------------------\n", - "CPLE_OpenFailedError Traceback (most recent call last)\n", - "File rasterio/_base.pyx:310, in rasterio._base.DatasetBase.__init__()\n", - "\n", - "File rasterio/_base.pyx:221, in rasterio._base.open_dataset()\n", - "\n", - "File rasterio/_err.pyx:221, in rasterio._err.exc_wrap_pointer()\n", - "\n", - "CPLE_OpenFailedError: ./input_03335500/results_03335500/clipped_1_03335500.tif: No such file or directory\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "RasterioIOError Traceback (most recent call last)\n", - "Cell In[18], line 1\n", - "----> 1 with rasterio.open(clipped_raster_filename) as src1:\n", - " 2 fig, ax = plt.subplots(figsize = (8,8))\n", - " 3 mapped=rasterio.plot.show(src1,\n", - " 4 ax=ax, \n", - " 5 cmap='viridis')\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/rasterio/env.py:451, in ensure_env_with_credentials.<locals>.wrapper(*args, **kwds)\n", - " 448 session = DummySession()\n", - " 450 with env_ctor(session=session):\n", - "--> 451 return f(*args, **kwds)\n", - "\n", - "File /opt/conda/lib/python3.11/site-packages/rasterio/__init__.py:304, in open(fp, mode, driver, width, height, count, crs, transform, dtype, nodata, sharing, **kwargs)\n", - " 301 path = _parse_path(raw_dataset_path)\n", - " 303 if mode == \"r\":\n", - "--> 304 dataset = DatasetReader(path, driver=driver, sharing=sharing, **kwargs)\n", - " 305 elif mode == \"r+\":\n", - " 306 dataset = get_writer_for_path(path, driver=driver)(\n", - " 307 path, mode, driver=driver, sharing=sharing, **kwargs\n", - " 308 )\n", - "\n", - "File rasterio/_base.pyx:312, in rasterio._base.DatasetBase.__init__()\n", - "\n", - "RasterioIOError: ./input_03335500/results_03335500/clipped_1_03335500.tif: No such file or directory\n", - "</pre></div>" - ], - "text/plain": [ - "<IPython.core.display.HTML object>" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "with rasterio.open(clipped_raster_filename) as src1:\n", " fig, ax = plt.subplots(figsize = (8,8))\n", @@ -1613,7 +860,7 @@ ], "metadata": { "kernelspec": { - "display_name": "airavata_examples", + "display_name": "airavata", "language": "python", "name": "python3" },
