uranusjr commented on code in PR #31749:
URL: https://github.com/apache/airflow/pull/31749#discussion_r1221067840
##########
airflow/providers/amazon/aws/sensors/s3.py:
##########
@@ -93,54 +95,102 @@ def __init__(
self.check_fn = check_fn
self.aws_conn_id = aws_conn_id
self.verify = verify
-
- def _check_key(self, key):
- bucket_name, key = S3Hook.get_s3_bucket_key(self.bucket_name, key,
"bucket_name", "bucket_key")
- self.log.info("Poking for key : s3://%s/%s", bucket_name, key)
-
- """
- Set variable `files` which contains a list of dict which contains only
the size
- If needed we might want to add other attributes later
- Format: [{
- 'Size': int
- }]
- """
- if self.wildcard_match:
- prefix = re.split(r"[\[\*\?]", key, 1)[0]
- keys = self.hook.get_file_metadata(prefix, bucket_name)
- key_matches = [k for k in keys if fnmatch.fnmatch(k["Key"], key)]
- if len(key_matches) == 0:
- return False
-
- # Reduce the set of metadata to size only
- files = list(map(lambda f: {"Size": f["Size"]}, key_matches))
+ self.deferrable = deferrable
+
+ def execute(self, context: Context) -> Any:
+ if self.deferrable:
+ from airflow.providers.amazon.aws.triggers.s3 import S3KeyTrigger
+
+ self.defer(
+ trigger=S3KeyTrigger(
+ bucket_name=self.bucket_name,
+ bucket_key=self.bucket_key,
+ wildcard_match=self.wildcard_match,
+ aws_conn_id=self.aws_conn_id,
+ verify=self.verify,
+ ),
+ method_name="execute_complete",
+ )
else:
- obj = self.hook.head_object(key, bucket_name)
- if obj is None:
- return False
- files = [{"Size": obj["ContentLength"]}]
-
- if self.check_fn is not None:
- return self.check_fn(files)
-
- return True
+ super().execute(context=context)
def poke(self, context: Context):
if isinstance(self.bucket_key, str):
- return self._check_key(self.bucket_key)
+ self.bucket_keys = [self.bucket_key]
else:
- return all(self._check_key(key) for key in self.bucket_key)
+ self.bucket_keys = self.bucket_key
+ wildcard_keys = []
+ objs = []
+ bucket_key_names = []
+ for i in range(len(self.bucket_keys)):
+ bucket_key_names.append(
+ S3Hook.get_s3_bucket_key(self.bucket_name,
self.bucket_keys[i], "bucket_name", "bucket_key")
+ )
+ bucket_name = bucket_key_names[i][0]
+ key = bucket_key_names[i][1]
+ self.log.info("Poking for key : s3://%s/%s", bucket_name, key)
+ if self.wildcard_match:
+ prefix = re.split(r"[\[\*\?]", key, 1)[0]
+ wildcard_keys.append(self.hook.get_file_metadata(prefix,
bucket_name))
+ else:
+ objs.append(self.hook.head_object(key, bucket_name))
+
+ results = process_files(
+ self.bucket_keys, self.wildcard_match, wildcard_keys, objs,
self.check_fn, bucket_key_names
+ )[0]
+ return all(results)
@deprecated(reason="use `hook` property instead.")
def get_hook(self) -> S3Hook:
"""Create and return an S3Hook."""
return self.hook
+ def execute_complete(self, context, event=None):
+ self.log.info("Inside execute complete")
+ if event["status"] != "success":
+ raise AirflowException(f"Error: {event}")
+ else:
+ results = []
+ self.log.info("Success: %s", event)
+ if self.check_fn is not None:
+ for files in event["files_list"]:
+ results.append(self.check_fn(files))
+ return all(results)
Review Comment:
Why build the entire `results` list and then call `all()`? Instead you can
just do
```python
for f in event["files_list"]:
if not self.check_fn(f):
return False
return True
```
or
```python
return all(self.check_fn(f) for f in event["files_list"])
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]