malthe commented on a change in pull request #21686:
URL: https://github.com/apache/airflow/pull/21686#discussion_r810589242
##########
File path: airflow/providers/amazon/aws/hooks/lambda_function.py
##########
@@ -40,30 +40,31 @@ class LambdaHook(AwsBaseHook):
def __init__(
self,
- function_name: str,
- log_type: str = 'None',
- qualifier: str = '$LATEST',
- invocation_type: str = 'RequestResponse',
*args,
**kwargs,
) -> None:
- self.function_name = function_name
- self.log_type = log_type
- self.invocation_type = invocation_type
- self.qualifier = qualifier
kwargs["client_type"] = "lambda"
super().__init__(*args, **kwargs)
- def invoke_lambda(self, payload: str) -> str:
+ def invoke_lambda(self, function_name: str, **kwargs):
Review comment:
It would be better if the parameters were spelled out, especially
because they now also have typing information. But previously, those parameters
also had defaults.
##########
File path: airflow/providers/amazon/aws/operators/aws_lambda.py
##########
@@ -0,0 +1,121 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+from typing import TYPE_CHECKING, Optional, Sequence
+
+from airflow.models import BaseOperator
+from airflow.providers.amazon.aws.hooks.lambda_function import LambdaHook
+
+if TYPE_CHECKING:
+ from airflow.utils.context import Context
+
+
+class AwsLambdaInvokeFunctionOperator(BaseOperator):
+ """
+ Invokes an AWS Lambda function.
+ You can invoke a function synchronously (and wait for the response),
+ or asynchronously.
+ To invoke a function asynchronously,
+ set InvocationType to `Event`.
+
+ :param function_name: The name of the AWS Lambda function, version, or
alias.
+ :param payload: The JSON string that you want to provide to your Lambda
function as input.
+ :param log_type: Set to Tail to include the execution log in the response.
Otherwise, set to "None".
+ :param qualifier: Specify a version or alias to invoke a published version
of the function.
+ :param aws_conn_id: The AWS connection ID to use
+ """
+
+ template_fields: Sequence[str] = ('function_name', 'payload', 'qualifier',
'invocation_type')
+ ui_color = '#ff7300'
+
+ def __init__(
+ self,
+ *,
+ function_name: str,
+ log_type: Optional[str] = None,
+ qualifier: Optional[str] = None,
+ invocation_type: Optional[str] = None,
+ client_context: Optional[str] = None,
+ payload: Optional[str] = None,
+ aws_conn_id='aws_default',
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self.FunctionName = function_name
Review comment:
This should be `function_name`, etc.
##########
File path: airflow/providers/amazon/aws/operators/aws_lambda.py
##########
@@ -0,0 +1,121 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+from typing import TYPE_CHECKING, Optional, Sequence
+
+from airflow.models import BaseOperator
+from airflow.providers.amazon.aws.hooks.lambda_function import LambdaHook
+
+if TYPE_CHECKING:
+ from airflow.utils.context import Context
+
+
+class AwsLambdaInvokeFunctionOperator(BaseOperator):
+ """
+ Invokes an AWS Lambda function.
+ You can invoke a function synchronously (and wait for the response),
+ or asynchronously.
+ To invoke a function asynchronously,
+ set InvocationType to `Event`.
+
+ :param function_name: The name of the AWS Lambda function, version, or
alias.
+ :param payload: The JSON string that you want to provide to your Lambda
function as input.
+ :param log_type: Set to Tail to include the execution log in the response.
Otherwise, set to "None".
+ :param qualifier: Specify a version or alias to invoke a published version
of the function.
+ :param aws_conn_id: The AWS connection ID to use
+ """
+
+ template_fields: Sequence[str] = ('function_name', 'payload', 'qualifier',
'invocation_type')
+ ui_color = '#ff7300'
+
+ def __init__(
+ self,
+ *,
+ function_name: str,
+ log_type: Optional[str] = None,
+ qualifier: Optional[str] = None,
+ invocation_type: Optional[str] = None,
+ client_context: Optional[str] = None,
+ payload: Optional[str] = None,
+ aws_conn_id='aws_default',
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self.FunctionName = function_name
+ self.Payload = payload
+ self.LogType = log_type
+ self.Qualifier = qualifier
+ self.InvocationType = invocation_type
+ self.ClientContext = client_context
+ self.aws_conn_id = aws_conn_id
+
+ def execute(self, context: 'Context'):
+ """
+ Invokes the target AWS Lambda function from Airflow.
+
+ :return: The response payload from the function, or an error object.
+ """
+ hook = LambdaHook(aws_conn_id=self.aws_conn_id)
+ success_status_codes = [200, 202, 204]
+ self.log.info("Invoking AWS Lambda function: %s with payload: %s",
self.FunctionName, self.Payload)
+ try:
+ response = hook.invoke_lambda(
+ function_name=self.FunctionName,
+ **{
+ key: self.__dict__[key]
Review comment:
This should be refactored a bit and the `kwargs` extraction can be put
outside of the `try` handler.
##########
File path: airflow/providers/amazon/aws/hooks/lambda_function.py
##########
@@ -40,30 +40,31 @@ class LambdaHook(AwsBaseHook):
def __init__(
self,
- function_name: str,
- log_type: str = 'None',
- qualifier: str = '$LATEST',
- invocation_type: str = 'RequestResponse',
*args,
**kwargs,
) -> None:
- self.function_name = function_name
- self.log_type = log_type
- self.invocation_type = invocation_type
- self.qualifier = qualifier
kwargs["client_type"] = "lambda"
super().__init__(*args, **kwargs)
- def invoke_lambda(self, payload: str) -> str:
+ def invoke_lambda(self, function_name: str, **kwargs):
"""Invoke Lambda Function"""
response = self.conn.invoke(
- FunctionName=self.function_name,
- InvocationType=self.invocation_type,
- LogType=self.log_type,
- Payload=payload,
- Qualifier=self.qualifier,
+ FunctionName=function_name, **{k: v for k, v in kwargs.items() if
v is not None}
)
+ return response
+ def create_lambda(
+ self, function_name: str, runtime: str, role: str, handler: str, code:
dict, **kwargs
+ ) -> dict:
+ """Create a Lambda Function"""
+ response = self.conn.create_function(
+ FunctionName=function_name,
+ Runtime=runtime,
+ Role=role,
+ Handler=handler,
+ Code=code,
+ **{k: v for k, v in kwargs.items() if v is not None},
Review comment:
What are these `kwargs` – the method is easier to use if they're spelled
out to the user.
##########
File path: airflow/providers/amazon/aws/operators/aws_lambda.py
##########
@@ -0,0 +1,121 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+from typing import TYPE_CHECKING, Optional, Sequence
+
+from airflow.models import BaseOperator
+from airflow.providers.amazon.aws.hooks.lambda_function import LambdaHook
+
+if TYPE_CHECKING:
+ from airflow.utils.context import Context
+
+
+class AwsLambdaInvokeFunctionOperator(BaseOperator):
+ """
+ Invokes an AWS Lambda function.
+ You can invoke a function synchronously (and wait for the response),
+ or asynchronously.
+ To invoke a function asynchronously,
+ set InvocationType to `Event`.
+
+ :param function_name: The name of the AWS Lambda function, version, or
alias.
+ :param payload: The JSON string that you want to provide to your Lambda
function as input.
+ :param log_type: Set to Tail to include the execution log in the response.
Otherwise, set to "None".
+ :param qualifier: Specify a version or alias to invoke a published version
of the function.
+ :param aws_conn_id: The AWS connection ID to use
+ """
+
+ template_fields: Sequence[str] = ('function_name', 'payload', 'qualifier',
'invocation_type')
+ ui_color = '#ff7300'
+
+ def __init__(
+ self,
+ *,
+ function_name: str,
+ log_type: Optional[str] = None,
+ qualifier: Optional[str] = None,
+ invocation_type: Optional[str] = None,
+ client_context: Optional[str] = None,
+ payload: Optional[str] = None,
+ aws_conn_id='aws_default',
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self.FunctionName = function_name
+ self.Payload = payload
+ self.LogType = log_type
+ self.Qualifier = qualifier
+ self.InvocationType = invocation_type
+ self.ClientContext = client_context
+ self.aws_conn_id = aws_conn_id
+
+ def execute(self, context: 'Context'):
+ """
+ Invokes the target AWS Lambda function from Airflow.
+
+ :return: The response payload from the function, or an error object.
+ """
+ hook = LambdaHook(aws_conn_id=self.aws_conn_id)
+ success_status_codes = [200, 202, 204]
+ self.log.info("Invoking AWS Lambda function: %s with payload: %s",
self.FunctionName, self.Payload)
+ try:
+ response = hook.invoke_lambda(
+ function_name=self.FunctionName,
+ **{
+ key: self.__dict__[key]
+ for key in self.__dict__
+ if (
+ key
+ in [
+ "Payload",
+ "LogType",
+ "Qualifier",
+ "InvocationType",
+ "ClientContext",
+ ]
+ and self.__dict__[key] is not None
+ )
+ },
+ )
+ self.log.info("Lambda response metadata: %s",
json.dumps(response.get("ResponseMetadata")))
Review comment:
I don't think you need to serialize this as JSON; a Python repr string
would be fine here:
```python
self.log.info("Lambda response metadata: %r",
response.get("ResponseMetadata"))
```
Also, there's a lot of repetition in the error handling here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
