edwardwang888 commented on a change in pull request #21686: URL: https://github.com/apache/airflow/pull/21686#discussion_r810689118
########## File path: tests/providers/amazon/aws/operators/test_lambda.py ########## @@ -0,0 +1,107 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import io +import json +import unittest +import zipfile + +from moto import mock_iam, mock_lambda, mock_sts + +from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook +from airflow.providers.amazon.aws.hooks.lambda_function import LambdaHook +from airflow.providers.amazon.aws.operators.aws_lambda import AwsLambdaInvokeFunctionOperator + + +@mock_lambda +@mock_sts +@mock_iam +class TestAwsLambdaInvokeFunctionOperator(unittest.TestCase): + def test_init(self): + lambda_operator = AwsLambdaInvokeFunctionOperator( + task_id="test", + function_name="test", + payload=json.dumps({"TestInput": "Testdata"}), + log_type="None", + aws_conn_id="aws_conn_test", + ) + assert lambda_operator.task_id == "test" + assert lambda_operator.function_name == "test" + assert lambda_operator.payload == json.dumps({"TestInput": "Testdata"}) + assert lambda_operator.log_type == "None" + assert lambda_operator.aws_conn_id == "aws_conn_test" + + def create_zip(self, body): + code = body + zip_output = io.BytesIO() + zip_file = zipfile.ZipFile(zip_output, "w", zipfile.ZIP_DEFLATED) + zip_file.writestr("lambda_function.py", code) + zip_file.close() Review comment: `ZipFile` can be used as a context manager: ``` with zipfile.ZipFile(...) as zip_file: zip_file.write_str(...) ``` ########## File path: airflow/providers/amazon/aws/operators/aws_lambda.py ########## @@ -0,0 +1,103 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +from typing import TYPE_CHECKING, Optional, Sequence + +from airflow.models import BaseOperator +from airflow.providers.amazon.aws.hooks.lambda_function import LambdaHook + +if TYPE_CHECKING: + from airflow.utils.context import Context + + +class AwsLambdaInvokeFunctionOperator(BaseOperator): + """ + Invokes an AWS Lambda function. + You can invoke a function synchronously (and wait for the response), + or asynchronously. + To invoke a function asynchronously, + set `invocation_type` to `Event`. For more details, + review the boto3 Lambda invoke docs. + + :param function_name: The name of the AWS Lambda function, version, or alias. + :param payload: The JSON string that you want to provide to your Lambda function as input. + :param log_type: Set to Tail to include the execution log in the response. Otherwise, set to "None". + :param qualifier: Specify a version or alias to invoke a published version of the function. + :param aws_conn_id: The AWS connection ID to use + """ + + template_fields: Sequence[str] = ('function_name', 'payload', 'qualifier', 'invocation_type') + ui_color = '#ff7300' + + def __init__( + self, + *, + function_name: str, + log_type: Optional[str] = None, + qualifier: Optional[str] = None, + invocation_type: Optional[str] = None, + client_context: Optional[str] = None, + payload: Optional[str] = None, + aws_conn_id='aws_default', + **kwargs, + ): + super().__init__(**kwargs) + self.function_name = function_name + self.payload = payload + self.log_type = log_type + self.qualifier = qualifier + self.invocation_type = invocation_type + self.client_context = client_context + self.aws_conn_id = aws_conn_id + + def execute(self, context: 'Context'): + """ + Invokes the target AWS Lambda function from Airflow. + + :return: The response payload from the function, or an error object. + """ + hook = LambdaHook(aws_conn_id=self.aws_conn_id) + success_status_codes = [200, 202, 204] + self.log.info("Invoking AWS Lambda function: %s with payload: %s", self.function_name, self.payload) + try: + response = hook.invoke_lambda( + function_name=self.function_name, + invocation_type=self.invocation_type, + log_type=self.log_type, + client_context=self.client_context, + payload=self.payload, + qualifier=self.qualifier, + ) + self.log.info("Lambda response metadata: %r", response.get("ResponseMetadata")) + except Exception as e: + self.log.error(e) + raise e Review comment: This `except` block doesn't seem needed if you are simply re-printing the error? ########## File path: tests/providers/amazon/aws/operators/test_lambda.py ########## @@ -0,0 +1,107 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import io +import json +import unittest +import zipfile + +from moto import mock_iam, mock_lambda, mock_sts + +from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook +from airflow.providers.amazon.aws.hooks.lambda_function import LambdaHook +from airflow.providers.amazon.aws.operators.aws_lambda import AwsLambdaInvokeFunctionOperator + + +@mock_lambda +@mock_sts +@mock_iam +class TestAwsLambdaInvokeFunctionOperator(unittest.TestCase): + def test_init(self): + lambda_operator = AwsLambdaInvokeFunctionOperator( + task_id="test", + function_name="test", + payload=json.dumps({"TestInput": "Testdata"}), + log_type="None", + aws_conn_id="aws_conn_test", + ) + assert lambda_operator.task_id == "test" + assert lambda_operator.function_name == "test" + assert lambda_operator.payload == json.dumps({"TestInput": "Testdata"}) + assert lambda_operator.log_type == "None" + assert lambda_operator.aws_conn_id == "aws_conn_test" + + def create_zip(self, body): + code = body + zip_output = io.BytesIO() + zip_file = zipfile.ZipFile(zip_output, "w", zipfile.ZIP_DEFLATED) + zip_file.writestr("lambda_function.py", code) + zip_file.close() + zip_output.seek(0) + return zip_output.read() + + def create_iam_role(self, role_name: str): + iam = AwsBaseHook("aws_conn_test", client_type="iam") + resp = iam.conn.create_role( + RoleName=role_name, + AssumeRolePolicyDocument=json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"Service": "lambda.amazonaws.com"}, + "Action": "sts:AssumeRole", + } + ], + } + ), + Description="IAM role for Lambda execution.", + ) + return resp["Role"]["Arn"] + + def create_lambda_function(self, function_name: str): Review comment: I think the helper functions (`create_zip`, `create_iam_role`, `create_lambda_function`) can be made static methods because they are not really instance-specific? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
