feluelle commented on a change in pull request #9350:
URL: https://github.com/apache/airflow/pull/9350#discussion_r442663134
##########
File path: airflow/providers/amazon/aws/operators/s3_delete_objects.py
##########
@@ -56,22 +59,31 @@ class S3DeleteObjectsOperator(BaseOperator):
:type verify: bool or str
"""
- template_fields = ('keys', 'bucket')
+ template_fields = ('keys', 'bucket', 'prefix')
@apply_defaults
def __init__(
self,
bucket,
- keys,
+ keys=None,
+ prefix=None,
aws_conn_id='aws_default',
verify=None,
*args, **kwargs):
+
+ if not bool(keys) ^ bool(prefix):
Review comment:
Oh ya - correct. 👍 We want to **not** have both be None or set at the
same time to raise an error.
##########
File path: airflow/providers/amazon/aws/operators/s3_delete_objects.py
##########
@@ -56,22 +59,31 @@ class S3DeleteObjectsOperator(BaseOperator):
:type verify: bool or str
"""
- template_fields = ('keys', 'bucket')
+ template_fields = ('keys', 'bucket', 'prefix')
@apply_defaults
def __init__(
self,
bucket,
- keys,
+ keys=None,
+ prefix=None,
aws_conn_id='aws_default',
verify=None,
*args, **kwargs):
+
+ if not bool(keys) ^ bool(prefix):
+ raise ValueError("Either keys or prefix should be set.")
+
super().__init__(*args, **kwargs)
self.bucket = bucket
self.keys = keys
+ self.prefix = prefix
self.aws_conn_id = aws_conn_id
self.verify = verify
def execute(self, context):
s3_hook = S3Hook(aws_conn_id=self.aws_conn_id, verify=self.verify)
- s3_hook.delete_objects(bucket=self.bucket, keys=self.keys)
+
+ keys = self.keys or s3_hook.list_keys(bucket_name=self.bucket,
prefix=self.prefix)
+ if keys:
Review comment:
Good catch! 👍
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
