ashb closed pull request #3172: [AIRFLOW-2216] Use profile for AWS hook if S3
config file provided in aws_default connection extra parameters
URL: https://github.com/apache/incubator-airflow/pull/3172
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/airflow/contrib/hooks/aws_hook.py
b/airflow/contrib/hooks/aws_hook.py
index 2a8fa5f823..e4020fd35b 100644
--- a/airflow/contrib/hooks/aws_hook.py
+++ b/airflow/contrib/hooks/aws_hook.py
@@ -100,8 +100,11 @@ def _get_credentials(self, region_name):
elif 's3_config_file' in connection_object.extra_dejson:
aws_access_key_id, aws_secret_access_key = \
-
_parse_s3_config(connection_object.extra_dejson['s3_config_file'],
-
connection_object.extra_dejson.get('s3_config_format'))
+ _parse_s3_config(
+ connection_object.extra_dejson['s3_config_file'],
+ connection_object.extra_dejson['s3_config_format'],
+ connection_object.extra_dejson['profile']
+ )
if region_name is None:
region_name =
connection_object.extra_dejson.get('region_name')
diff --git a/tests/contrib/hooks/test_aws_hook.py
b/tests/contrib/hooks/test_aws_hook.py
index 086e486144..dd1b69e173 100644
--- a/tests/contrib/hooks/test_aws_hook.py
+++ b/tests/contrib/hooks/test_aws_hook.py
@@ -14,6 +14,7 @@
#
import unittest
+
import boto3
from airflow import configuration
@@ -141,6 +142,26 @@ def test_get_credentials_from_extra(self,
mock_get_connection):
self.assertEqual(credentials_from_hook.secret_key,
'aws_secret_access_key')
self.assertIsNone(credentials_from_hook.token)
+ @mock.patch('airflow.contrib.hooks.aws_hook._parse_s3_config',
+ return_value=('aws_access_key_id', 'aws_secret_access_key'))
+ @mock.patch.object(AwsHook, 'get_connection')
+ def test_get_credentials_from_extra_with_s3_config_and_profile(
+ self, mock_get_connection, mock_parse_s3_config
+ ):
+ mock_connection = Connection(
+ extra='{"s3_config_format": "aws", '
+ '"profile": "test", '
+ '"s3_config_file": "aws-credentials", '
+ '"region_name": "us-east-1"}')
+ mock_get_connection.return_value = mock_connection
+ hook = AwsHook()
+ hook._get_credentials(region_name=None)
+ mock_parse_s3_config.assert_called_with(
+ 'aws-credentials',
+ 'aws',
+ 'test'
+ )
+
@unittest.skipIf(mock_sts is None, 'mock_sts package not present')
@mock.patch.object(AwsHook, 'get_connection')
@mock_sts
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
