kanga333 commented on a change in pull request #9350:
URL: https://github.com/apache/airflow/pull/9350#discussion_r442284808
##########
File path: airflow/providers/amazon/aws/operators/s3_delete_objects.py
##########
@@ -56,22 +59,34 @@ class S3DeleteObjectsOperator(BaseOperator):
:type verify: bool or str
"""
- template_fields = ('keys', 'bucket')
+ template_fields = ('keys', 'bucket', 'prefix')
@apply_defaults
def __init__(
self,
bucket,
- keys,
+ keys=None,
+ prefix=None,
aws_conn_id='aws_default',
verify=None,
*args, **kwargs):
+
+ if not keys and not prefix:
+ raise ValueError("Either keys or prefix should be set. Both are
None")
+
super().__init__(*args, **kwargs)
self.bucket = bucket
self.keys = keys
+ self.prefix = prefix
self.aws_conn_id = aws_conn_id
self.verify = verify
def execute(self, context):
s3_hook = S3Hook(aws_conn_id=self.aws_conn_id, verify=self.verify)
- s3_hook.delete_objects(bucket=self.bucket, keys=self.keys)
+
+ if self.keys:
+ keys = self.keys
+ else:
+ keys = s3_hook.list_keys(bucket_name=self.bucket,
prefix=self.prefix)
Review comment:
Passing an empty list to the S3DeleteObjectsOperator keys in the current
implementation does not raise an exception. We had better not change this
behavior.
```python
keys = []
# Nothing is done. No exceptions will be made.
S3DeleteObjectsOperator(task_id=task_id, bucket=bucket, keys=keys)
```
If the argument is `keys=[]` and `prefix=None`, then the following code will
execute `list_keys` with `prefix=None` and raise an exception.
```python
keys = self.keys or s3_hook.list_keys(bucket_name=self.bucket,
prefix=self.prefix)
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
