[
https://issues.apache.org/jira/browse/AIRFLOW-2216?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16677307#comment-16677307
]
ASF GitHub Bot commented on AIRFLOW-2216:
-----------------------------------------
ashb closed pull request #3172: [AIRFLOW-2216] Use profile for AWS hook if S3
config file provided in aws_default connection extra parameters
URL: https://github.com/apache/incubator-airflow/pull/3172
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/airflow/contrib/hooks/aws_hook.py
b/airflow/contrib/hooks/aws_hook.py
index 2a8fa5f823..e4020fd35b 100644
--- a/airflow/contrib/hooks/aws_hook.py
+++ b/airflow/contrib/hooks/aws_hook.py
@@ -100,8 +100,11 @@ def _get_credentials(self, region_name):
elif 's3_config_file' in connection_object.extra_dejson:
aws_access_key_id, aws_secret_access_key = \
-
_parse_s3_config(connection_object.extra_dejson['s3_config_file'],
-
connection_object.extra_dejson.get('s3_config_format'))
+ _parse_s3_config(
+ connection_object.extra_dejson['s3_config_file'],
+ connection_object.extra_dejson['s3_config_format'],
+ connection_object.extra_dejson['profile']
+ )
if region_name is None:
region_name =
connection_object.extra_dejson.get('region_name')
diff --git a/tests/contrib/hooks/test_aws_hook.py
b/tests/contrib/hooks/test_aws_hook.py
index 086e486144..dd1b69e173 100644
--- a/tests/contrib/hooks/test_aws_hook.py
+++ b/tests/contrib/hooks/test_aws_hook.py
@@ -14,6 +14,7 @@
#
import unittest
+
import boto3
from airflow import configuration
@@ -141,6 +142,26 @@ def test_get_credentials_from_extra(self,
mock_get_connection):
self.assertEqual(credentials_from_hook.secret_key,
'aws_secret_access_key')
self.assertIsNone(credentials_from_hook.token)
+ @mock.patch('airflow.contrib.hooks.aws_hook._parse_s3_config',
+ return_value=('aws_access_key_id', 'aws_secret_access_key'))
+ @mock.patch.object(AwsHook, 'get_connection')
+ def test_get_credentials_from_extra_with_s3_config_and_profile(
+ self, mock_get_connection, mock_parse_s3_config
+ ):
+ mock_connection = Connection(
+ extra='{"s3_config_format": "aws", '
+ '"profile": "test", '
+ '"s3_config_file": "aws-credentials", '
+ '"region_name": "us-east-1"}')
+ mock_get_connection.return_value = mock_connection
+ hook = AwsHook()
+ hook._get_credentials(region_name=None)
+ mock_parse_s3_config.assert_called_with(
+ 'aws-credentials',
+ 'aws',
+ 'test'
+ )
+
@unittest.skipIf(mock_sts is None, 'mock_sts package not present')
@mock.patch.object(AwsHook, 'get_connection')
@mock_sts
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> Cannot specify a profile for AWS Hook to load with s3 config file
> -----------------------------------------------------------------
>
> Key: AIRFLOW-2216
> URL: https://issues.apache.org/jira/browse/AIRFLOW-2216
> Project: Apache Airflow
> Issue Type: Bug
> Components: operators
> Affects Versions: 1.9.0
> Environment: IDE: PyCharm
> Airflow 1.9
> Python 3.4.3
> Reporter: Lorena Mesa
> Assignee: Lorena Mesa
> Priority: Minor
> Fix For: 1.10.1
>
>
> Currently the source code for AWS Hook doesn't permit the user to provide a
> profile when their aws connection object specifies in the extra param's
> information on s3_config_file:
> {code:java}
> def _get_credentials(self, region_name):
> aws_access_key_id = None
> aws_secret_access_key = None
> aws_session_token = None
> endpoint_url = None
> if self.aws_conn_id:
> try:
> # Cut for brevity
> elif 's3_config_file' in connection_object.extra_dejson:
> aws_access_key_id, aws_secret_access_key = \
> _parse_s3_config(connection_object.extra_dejson['s3_config_file'],
> connection_object.extra_dejson.get('s3_config_format'),
> connection_object.extra_dejson.get('profile')){code}
> The _parse_s3_config method has a param for profile set to none, so by not
> providing it in the method you cannot now specify a profile credential to be
> loaded.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
