ferruzzi commented on a change in pull request #16571: URL: https://github.com/apache/airflow/pull/16571#discussion_r660150440
########## File path: airflow/providers/amazon/aws/utils/eks_kube_config.py ########## @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import os +from shutil import which +from typing import Optional + +import boto3 +import yaml + +HOME = os.environ.get('HOME', '/tmp') +DEFAULT_KUBE_CONFIG_FILENAME = 'config' +DEFAULT_KUBE_CONFIG_PATH = str(os.path.join(HOME, '/.kube/', DEFAULT_KUBE_CONFIG_FILENAME)) +DEFAULT_CONTEXT_NAME = 'aws' +DEFAULT_NAMESPACE_NAME = 'default' +DEFAULT_POD_USERNAME = 'aws' + + +def generate_config_file( + eks_cluster_name: str, + eks_namespace_name: str, + aws_profile: Optional[str], + kube_config_file_location: Optional[str] = DEFAULT_KUBE_CONFIG_PATH, + pod_username: Optional[str] = DEFAULT_POD_USERNAME, + pod_context: Optional[str] = DEFAULT_CONTEXT_NAME, + role_arn: Optional[str] = None, + aws_region: Optional[str] = None, +) -> None: + """ + Writes the kubeconfig file given an EKS Cluster name, AWS region, and file path. + + :param eks_cluster_name: The name of the cluster to create the EKS Managed Nodegroup in. + :type eks_cluster_name: str + :param eks_namespace_name: The namespace to run within kubernetes. + :type eks_namespace_name: str + :param aws_profile: The named profile containing the credentials for the AWS CLI tool to use. + :type aws_profile: str + :param kube_config_file_location: Path to save the generated kube_config file to. + :type kube_config_file_location: str + :param pod_username: The username under which to execute the pod. + :type pod_username: str + :param pod_context: The name of the context access parameters to use. + :type pod_context: str + :param role_arn: The Amazon Resource Name (ARN) of the IAM role to associate with your nodegroup. + :type role_arn: str + :param aws_region: The name of the AWS Region the EKS Cluster resides in. + :type aws_region: str + """ + installed = which("aws") + if installed is None: + message = ( + "AWS CLI version 2 must be installed on the worker. See: " + "https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html"; + ) + print(message) + raise UnmetDependency(message) + + # Set up the client + session = boto3.Session(region_name=aws_region, profile_name=aws_profile) + eks_client = session.client("eks") + + # get cluster details + cluster = eks_client.describe_cluster(name=eks_cluster_name) + cluster_cert = cluster["cluster"]["certificateAuthority"]["data"] + cluster_ep = cluster["cluster"]["endpoint"] + + # build the cluster config hash + cli_args = [ + "--region", + aws_region, + "eks", + "get-token", + "--cluster-name", + eks_cluster_name, + ] + if role_arn: + cli_args.extend(["--role-arn", role_arn]) Review comment: Using the AWS CLI was not our preference either, but had to settle for this due to other options not working correctly. The EKS service does not support providing the token directly at all. Searching online came up with a few workarounds, but the lesser evil was assuming that someone using an AWS service would be able to install an AWS tool. There is also precedent for it in [Google's Kubernetes Operators](https://github.com/apache/airflow/blob/2625007c8aeca9ed98dea361ba13c2622482d71f/airflow/providers/google/cloud/operators/kubernetes_engine.py#L319) using the gcloud local script to accomplish the same thing. I'll test using your provided example to see if that works and report back. Dropping that dependency would be great. ########## File path: docs/apache-airflow-providers-amazon/operators/eks.rst ########## @@ -0,0 +1,265 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +Amazon Elastic Kubernetes Service (EKS) Operators +================================================= + +`Amazon Elastic Kubernetes Service (Amazon EKS) <https://aws.amazon.com/eks/>`__ is a managed service +that makes it easy for you to run Kubernetes on AWS without needing to stand up or maintain your own +Kubernetes control plane. Kubernetes is an open-source system for automating the deployment, scaling, +and management of containerized applications. + +.. contents:: + :depth: 1 + :local: + +Prerequisite Tasks +------------------ + +.. include:: _partials/prerequisite_tasks.rst + +Overview +-------- + +Airflow to Amazon Elastic Kubernetes Service (EKS) integration provides Operators to create and +interact with the EKS clusters and compute infrastructure. + + - :class:`~airflow.providers.amazon.aws.operators.eks` + +4 example_dags are provided which showcase these operators in action. + + - example_eks_create_cluster.py + - example_eks_create_cluster_with_nodegroup.py + - example_eks_create_nodegroup.py + - example_eks_pod_operator.py + + +.. _howto/operator:EKSCreateClusterOperator: + +Creating Amazon EKS Clusters +---------------------------- + +Purpose +""""""" + +This example dag ``example_eks_create_cluster.py`` uses ``EKSCreateClusterOperator`` to create an Amazon +EKS Cluster, ``EKSListClustersOperator`` and ``EKSDescribeClusterOperator`` to verify creation, then +``EKSDeleteClusterOperator`` to delete the Cluster. + +Prerequisites +""""""""""""" + +An AWS IAM role with the following permissions: + + "eks.amazonaws.com" must be added to the Trusted Relationships + "AmazonEKSClusterPolicy" IAM Policy must be attached + +Defining tasks +"""""""""""""" + +In the following code we create a new Amazon EKS Cluster. + +.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_eks_create_cluster.py + :language: python + :start-after: [START howto_operator_eks_create_cluster] + :end-before: [END howto_operator_eks_create_cluster] + + +.. _howto/operator:EKSListClustersOperator: +.. _howto/operator:EKSDescribeClusterOperator: + + +Listing and Describing Amazon EKS Clusters +------------------------------------------- + +Defining tasks +"""""""""""""" + +In the following code we list all Amazon EKS Clusters. + +.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_eks_create_cluster.py + :language: python + :start-after: [START howto_operator_eks_list_clusters] + :end-before: [END howto_operator_eks_list_clusters] + +In the following code we retrieve details for a given Amazon EKS Cluster. + +.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_eks_create_cluster.py + :language: python + :start-after: [START howto_operator_eks_describe_cluster] + :end-before: [END howto_operator_eks_describe_cluster] + + +.. _howto/operator:EKSDeleteClusterOperator: + +Deleting Amazon EKS Clusters +---------------------------- + +Defining tasks +"""""""""""""" + +In the following code we delete a given Amazon EKS Cluster. + +.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_eks_create_cluster.py + :language: python + :start-after: [START howto_operator_eks_delete_cluster] + :end-before: [END howto_operator_eks_delete_cluster] + + +.. _howto/operator:EKSCreateNodegroupOperator: + +Creating Amazon EKS Managed NodeGroups +-------------------------------------- + +Purpose +""""""" + +This example dag ``example_eks_create_nodegroup.py`` uses ``EKSCreateNodegroupOperator`` +to create an Amazon EKS Managed Nodegroup using an existing cluster, ``EKSListNodegroupsOperator`` +and ``EKSDescribeNodegroupOperator`` to verify creation, then ``EKSDeleteNodegroupOperator`` +to delete the nodegroup. + +Prerequisites +""""""""""""" + +An AWS IAM role with the following permissions: + + "ec2.amazon.aws.com" must be in the Trusted Relationships + "AmazonEC2ContainerRegistryReadOnly" IAM Policy must be attached + "AmazonEKSWorkerNodePolicy" IAM Policy must be attached + +Defining tasks +"""""""""""""" + +In the following code we create a new Amazon EKS Managed Nodegroup. + +.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_eks_create_nodegroup.py + :language: python + :start-after: [START howto_operator_eks_create_nodegroup] + :end-before: [END howto_operator_eks_create_nodegroup] + + +.. _howto/operator:EKSListNodegroupsOperator: +.. _howto/operator:EKSDescribeNodegroupOperator: + +Listing and Describing Amazon EKS Clusters +------------------------------------------- + +Defining tasks +"""""""""""""" + +In the following code we retrieve details for a given Amazon EKS nodegroup. + +.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_eks_create_nodegroup.py + :language: python + :start-after: [START howto_operator_eks_describe_nodegroup] + :end-before: [END howto_operator_eks_describe_nodegroup] + + +In the following code we list all Amazon EKS Nodegroups in a given EKS Cluster. + +.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_eks_create_nodegroup.py + :language: python + :start-after: [START howto_operator_eks_list_nodegroup] + :end-before: [END howto_operator_eks_list_nodegroup] + + +.. _howto/operator:EKSDeleteNodegroupOperator: + +Deleting Amazon EKS Managed Nodegroups +-------------------------------------- + +Defining tasks +"""""""""""""" + +In the following code we delete an Amazon EKS nodegroup. + +.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_eks_create_nodegroup.py + :language: python + :start-after: [START howto_operator_eks_delete_nodegroup] + :end-before: [END howto_operator_eks_delete_nodegroup] + + +Creating Amazon EKS Clusters and Node Groups Together +------------------------------------------------------ + +Purpose +""""""" + +This example dag ``example_eks_create_stack.py`` demonstrates using +``EKSCreateClusterOperator`` to create an Amazon EKS cluster and underlying +Amazon EKS node group in one command. ``EKSDescribeClustersOperator`` and +``EKSDescribeNodegroupsOperator`` verify creation, then ``EKSDeleteClusterOperator`` +deletes all created resources. + +Prerequisites +""""""""""""" + + "ec2.amazon.aws.com" must be in the Trusted Relationships + "eks.amazonaws.com" must be added to the Trusted Relationships + "AmazonEC2ContainerRegistryReadOnly" IAM Policy must be attached + "AmazonEKSClusterPolicy" IAM Policy must be attached + "AmazonEKSWorkerNodePolicy" IAM Policy must be attached Review comment: Does the spell check know to ignore anything inside double back ticks? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
