Taragolis commented on code in PR #35037:
URL: https://github.com/apache/airflow/pull/35037#discussion_r1364704230
##########
airflow/providers/amazon/aws/hooks/s3.py:
##########
@@ -912,14 +912,27 @@ def get_key(self, key: str, bucket_name: str | None =
None) -> S3ResourceObject:
:param bucket_name: the name of the bucket
:return: the key object from the bucket
"""
+
+ def __sanitize_extra_args() -> dict[str, str]:
+ """Parse extra_args and return a dict with only the args listed in
ALLOWED_DOWNLOAD_ARGS."""
+ return {
+ arg_name: arg_value
+ for (arg_name, arg_value) in self.extra_args.items()
+ if arg_name in S3Transfer(self.conn).ALLOWED_DOWNLOAD_ARGS
+ }
+
s3_resource = self.get_session().resource(
"s3",
endpoint_url=self.conn_config.endpoint_url,
config=self.config,
verify=self.verify,
)
obj = s3_resource.Object(bucket_name, key)
- obj.load()
+
+ # TODO inline this after debugging
+ new_args = __sanitize_extra_args()
+
+ obj.load(**new_args)
Review Comment:
> Pretty sure base64 was one I tried
I checked, somewhere internal make it for you, so I grab this headers from
debug over failure call. I provide `aaaaa` as input for key, and that is my
botocore request headers
```python
{'x-amz-server-side-encryption-customer-key': 'YWFhYWE=',
'x-amz-server-side-encryption-customer-algorithm': 'AES256',
'x-amz-server-side-encryption-customer-key-MD5': 'redacted but also base64',
'User-Agent': 'Boto3/1.28.17 md/Botocore#1.31.17 ua/2.0 os/macos#22.6.0
md/arch#arm64 lang/python#3.9.10 md/pyimpl#CPython cfg/retry-mode#legacy
Botocore/1.31.17 Resource'}
```
The main limitation, that the key **should be exactly** 32 bytes long,
thanks SO for hint, but original sample doesn't work. So finally check it by
pure `boto3` and that works
```python
import os
import boto3
bucket = 'your-bucket'
key = 'sample.txt'
sse_customer_alg = "AES256"
sse_customer_key = b"a" * 32 # Should be 32 bytes long
# sse_customer_key = "b" * 32 # This also should work
# sse_customer_key = os.urandom(32) # This also should work
# sse_customer_key = "YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=" # This
one not working
# sse_customer_key = "¥" * 32 # This one not working, in bytes it is
greater than 32
session = boto3.session.Session(...) # required params
resource = session.resource(service_name="s3")
client = resource.meta.client
response = client.put_object(
Bucket=bucket,
Key=key,
Body=b"Awesome content!",
SSECustomerAlgorithm=sse_customer_alg,
SSECustomerKey=sse_customer_key,
)
obj = resource.Object(bucket, key)
obj.load(SSECustomerKey=sse_customer_key,
SSECustomerAlgorithm=sse_customer_alg)
result = obj.get(
SSECustomerKey=sse_customer_key, SSECustomerAlgorithm=sse_customer_alg
)['Body'].read().decode()
print(result)
# Awesome content!
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
