This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit 5a658fbac19cc623caf713664e7ad3b4e75f6455 Author: Marcus Christie <[email protected]> AuthorDate: Fri Nov 19 15:37:45 2021 -0500 AIRAVATA-3542 include and exclude patterns for multi experiment download --- airavata_django_portal_sdk/serializers.py | 13 ++++++++--- airavata_django_portal_sdk/views.py | 37 ++++++++++++++++++++++++------- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/airavata_django_portal_sdk/serializers.py b/airavata_django_portal_sdk/serializers.py index 2fc09b5..e86d010 100644 --- a/airavata_django_portal_sdk/serializers.py +++ b/airavata_django_portal_sdk/serializers.py @@ -1,12 +1,19 @@ from rest_framework import serializers -# class DownloadIncludeSerializer(serializers.Serializer): -# pattern = serializers.CharField() + +class FilenamePatternSerializer(serializers.Serializer): + pattern = serializers.CharField() + + +class IncludeFilenamePatternSerializer(FilenamePatternSerializer): + rename = serializers.CharField(required=False) class ExperimentDownloadSerializer(serializers.Serializer): experiment_id = serializers.CharField() - # includes = DownloadIncludeSerializer(many=True) + path = serializers.CharField(default="") + includes = IncludeFilenamePatternSerializer(many=True, required=False, default=None) + excludes = FilenamePatternSerializer(many=True, required=False, default=None) class MultiExperimentDownloadSerializer(serializers.Serializer): diff --git a/airavata_django_portal_sdk/views.py b/airavata_django_portal_sdk/views.py index 0592c2d..4c697b6 100644 --- a/airavata_django_portal_sdk/views.py +++ b/airavata_django_portal_sdk/views.py @@ -1,3 +1,4 @@ +import fnmatch import logging import os import tempfile @@ -121,9 +122,12 @@ def download_experiments(request, download_id=None): for experiment in experiments: experiment_id = experiment['experiment_id'] # Load experiment to make sure user has access to experiment - experiment = request.airavata_client.getExperiment(request.authz_token, experiment_id) - _add_experiment_directory_to_zipfile(request, zf, experiment_id, path="", - zipfile_prefix=get_valid_filename(experiment.experimentName)) + experiment_model = request.airavata_client.getExperiment(request.authz_token, experiment_id) + path = experiment['path'] + _add_experiment_directory_to_zipfile( + request, zf, experiment_id, path, + zipfile_prefix=os.path.join(get_valid_filename(experiment_model.experimentName), path), + includes=experiment['includes'], excludes=experiment['excludes']) filename = "experiments.zip" fp.seek(0) @@ -146,14 +150,31 @@ def _add_directory_to_zipfile(request, zf, path, directory=""): _add_directory_to_zipfile(request, zf, path, os.path.join(directory, d['name'])) -def _add_experiment_directory_to_zipfile(request, zf, experiment_id, path, directory="", zipfile_prefix=""): +def _add_experiment_directory_to_zipfile(request, zf, experiment_id, path, directory="", zipfile_prefix="", includes=None, excludes=None): directories, files = user_storage.list_experiment_dir(request, experiment_id, os.path.join(path, directory)) for file in files: - o = user_storage.open_file(request, data_product_uri=file['data-product-uri']) - zf.writestr(os.path.join(zipfile_prefix, directory, file['name']), o.read()) - if os.path.getsize(zf.filename) > MAX_DOWNLOAD_ZIPFILE_SIZE: - raise Exception(f"Zip file size exceeds max of {MAX_DOWNLOAD_ZIPFILE_SIZE} bytes") + matches = _matches_filters(file['name'], includes=includes, excludes=excludes) + if matches: + o = user_storage.open_file(request, data_product_uri=file['data-product-uri']) + zf.writestr(os.path.join(zipfile_prefix, directory, file['name']), o.read()) + if os.path.getsize(zf.filename) > MAX_DOWNLOAD_ZIPFILE_SIZE: + raise Exception(f"Zip file size exceeds max of {MAX_DOWNLOAD_ZIPFILE_SIZE} bytes") for d in directories: _add_experiment_directory_to_zipfile(request, zf, experiment_id, path, directory=os.path.join(directory, d['name']), zipfile_prefix=zipfile_prefix) + + +def _matches_filters(filename, includes=None, excludes=None): + # excludes take precedence + if excludes is not None and len(excludes) > 0: + for exclude in excludes: + if fnmatch.fnmatch(filename, exclude['pattern']): + return False + # if there are no include patterns, default to include all + if includes is None or len(includes) == 0: + return True + for include in includes: + if fnmatch.fnmatch(filename, include['pattern']): + return True + return False
