[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement
Azure Kubernetes Service Operator
URL: https://github.com/apache/airflow/pull/4530#discussion_r261851642
##
File path: airflow/contrib/utils/aks_utils.py
##
@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+import re
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.primitives import serialization
+
+
+def is_valid_ssh_rsa_public_key(openssh_pubkey):
+#
http://stackoverflow.com/questions/2494450/ssh-rsa-public-key-validation-using-a-regular-expression
+# A "good enough" check is to see if the key starts with the correct
header.
+import struct
+try:
+from base64 import decodebytes as base64_decode
+except ImportError:
+# deprecated and redirected to decodebytes in Python 3
+from base64 import decodestring as base64_decode
+
+parts = openssh_pubkey.split()
+if len(parts) < 2:
+return False
+key_type = parts[0]
+key_string = parts[1]
+
+data = base64_decode(key_string.encode()) #
pylint:disable=deprecated-method
+int_len = 4
+str_len = struct.unpack('>I', data[:int_len])[0] # this should return 7
+return data[int_len:int_len + str_len] == key_type.encode()
+
+
+def get_public_key(self):
+key = rsa.generate_private_key(backend=default_backend(),
public_exponent=65537, key_size=2048)
+return key.public_key().public_bytes(serialization.Encoding.OpenSSH,
+
serialization.PublicFormat.OpenSSH).decode('utf-8')
+
+
+def load_json(self, file_path):
+try:
+with open(file_path) as configFile:
+configData = json.load(configFile)
+except FileNotFoundError:
+print("Error: Expecting azurermconfig.json in current folder")
+raise
+return configData
+
+
+def get_poller_result(self, poller, wait=5):
+'''
+Consistent method of waiting on and retrieving results from Azure's long
poller
+:param poller Azure poller object
+:return object resulting from the original request
+'''
+try:
+delay = wait
+while not poller.done():
+self.log.info("Waiting for {0} sec".format(delay))
+poller.wait(timeout=delay)
+return poller.result()
+except Exception as exc:
+self.log.info("exception here {0} ".format(exc))
Review comment:
```suggestion
self.log.info("exception here %s", exc)
```
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r261851607 ## File path: airflow/contrib/operators/aks_operator.py ## @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.contrib.hooks.azure_kubernetes_hook import AzureKubernetesServiceHook +from airflow.models import BaseOperator +from airflow.utils.decorators import apply_defaults + +from azure.mgmt.containerservice.models import ContainerServiceLinuxProfile +from azure.mgmt.containerservice.models import ContainerServiceServicePrincipalProfile +from azure.mgmt.containerservice.models import ContainerServiceSshConfiguration +from azure.mgmt.containerservice.models import ContainerServiceSshPublicKey +from azure.mgmt.containerservice.models import ContainerServiceStorageProfileTypes +from azure.mgmt.containerservice.models import ManagedCluster +from azure.mgmt.containerservice.models import ManagedClusterAgentPoolProfile + +from airflow.contrib.utils.aks_utils import \ +is_valid_ssh_rsa_public_key, get_poller_result, get_public_key, get_default_dns_prefix +from msrestazure.azure_exceptions import CloudError + + +class AzureKubernetesOperator(BaseOperator): +""" +Start a Azure Kubernetes Service + +:param ci_conn_id: connection id of a +service principal which will be used to start the azure kubernetes service +:type ci_conn_id: str +:param resource_group: Required name of the resource group +wherein this container instance should be started +:type resource_group: str +:param name: Required name of this container. Please note this name +has to be unique in order to run containers in parallel. +:type name: str +:param ssh_key_value: the ssh value used to connect to machine to be used +:type ssh_key_value: str +:param dns_name_prefix: DNS prefix specified when creating the managed cluster. +:type region: dns_name_prefix +:param admin_username: The administrator username to use for Linux VMs. +:type admin_username: str +:param kubernetes_version: Version of Kubernetes specified when creating the managed cluster. +:type kubernetes_version: str +:param node_vm_size: Vm to be spin up. +:type node_vm_size: str or ContainerServiceVMSizeTypes Enum +:param node_osdisk_size: Size in GB to be used to specify the disk size for +every machine in this master/agent pool. If you specify 0, it will apply the default +osDisk size according to the vmSize specified. +:type node_osdisk_size: int +:param node_count: Number of agents (VMs) to host docker containers. +Allowed values must be in the range of 1 to 100 (inclusive). The default value is 1. +:type node_count: int +:param no_ssh_key: Specified if it is linuxprofile. +:type no_ssh_key: boolean +:param vnet_subnet_id: VNet SubnetID specifies the vnet's subnet identifier. +:type vnet_subnet_id: str +:param max_pods: Maximum number of pods that can run on a node. +:type max_pods: int +:param os_type: OsType to be used to specify os type. Choose from Linux and Windows. +Default to Linux. +:type os_type: str or OSType Enum +:param tags: Resource tags. +:type tags: dict[str, str] +:param location: Required resource location +:type location: str + +:Example: + +>>> a = AzureKubernetesOperator(task_id="task",ci_conn_id='azure_kubernetes_default', +resource_group="my_resource_group", +name="my_aks_container", +ssh_key_value=None, +dns_name_prefix=None, +location="my_region", +tags=None +) +""" +@apply_defaults +def __init__(self, ci_conn_id, resource_group, name, ssh_key_value, + dns_name_prefix=None, + location=None, + admin_username="azureuser", + kubernetes_version='', + node_vm_size="Standard_DS2_v2", + node_osdisk_size=0, + node_count=3, +
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement
Azure Kubernetes Service Operator
URL: https://github.com/apache/airflow/pull/4530#discussion_r261851638
##
File path: airflow/contrib/utils/aks_utils.py
##
@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+import re
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.primitives import serialization
+
+
+def is_valid_ssh_rsa_public_key(openssh_pubkey):
+#
http://stackoverflow.com/questions/2494450/ssh-rsa-public-key-validation-using-a-regular-expression
+# A "good enough" check is to see if the key starts with the correct
header.
+import struct
+try:
+from base64 import decodebytes as base64_decode
+except ImportError:
+# deprecated and redirected to decodebytes in Python 3
+from base64 import decodestring as base64_decode
+
+parts = openssh_pubkey.split()
+if len(parts) < 2:
+return False
+key_type = parts[0]
+key_string = parts[1]
+
+data = base64_decode(key_string.encode()) #
pylint:disable=deprecated-method
+int_len = 4
+str_len = struct.unpack('>I', data[:int_len])[0] # this should return 7
+return data[int_len:int_len + str_len] == key_type.encode()
+
+
+def get_public_key(self):
+key = rsa.generate_private_key(backend=default_backend(),
public_exponent=65537, key_size=2048)
+return key.public_key().public_bytes(serialization.Encoding.OpenSSH,
+
serialization.PublicFormat.OpenSSH).decode('utf-8')
+
+
+def load_json(self, file_path):
+try:
+with open(file_path) as configFile:
+configData = json.load(configFile)
+except FileNotFoundError:
+print("Error: Expecting azurermconfig.json in current folder")
+raise
+return configData
+
+
+def get_poller_result(self, poller, wait=5):
+'''
+Consistent method of waiting on and retrieving results from Azure's long
poller
+:param poller Azure poller object
+:return object resulting from the original request
+'''
+try:
+delay = wait
+while not poller.done():
+self.log.info("Waiting for {0} sec".format(delay))
Review comment:
```suggestion
self.log.info("Waiting for %s sec", delay)
```
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r261851626 ## File path: airflow/contrib/operators/aks_operator.py ## @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.contrib.hooks.azure_kubernetes_hook import AzureKubernetesServiceHook +from airflow.models import BaseOperator +from airflow.utils.decorators import apply_defaults + +from azure.mgmt.containerservice.models import ContainerServiceLinuxProfile +from azure.mgmt.containerservice.models import ContainerServiceServicePrincipalProfile +from azure.mgmt.containerservice.models import ContainerServiceSshConfiguration +from azure.mgmt.containerservice.models import ContainerServiceSshPublicKey +from azure.mgmt.containerservice.models import ContainerServiceStorageProfileTypes +from azure.mgmt.containerservice.models import ManagedCluster +from azure.mgmt.containerservice.models import ManagedClusterAgentPoolProfile + +from airflow.contrib.utils.aks_utils import \ +is_valid_ssh_rsa_public_key, get_poller_result, get_public_key, get_default_dns_prefix +from msrestazure.azure_exceptions import CloudError + + +class AzureKubernetesOperator(BaseOperator): +""" +Start a Azure Kubernetes Service + +:param ci_conn_id: connection id of a +service principal which will be used to start the azure kubernetes service +:type ci_conn_id: str +:param resource_group: Required name of the resource group +wherein this container instance should be started +:type resource_group: str +:param name: Required name of this container. Please note this name +has to be unique in order to run containers in parallel. +:type name: str +:param ssh_key_value: the ssh value used to connect to machine to be used +:type ssh_key_value: str +:param dns_name_prefix: DNS prefix specified when creating the managed cluster. +:type region: dns_name_prefix +:param admin_username: The administrator username to use for Linux VMs. +:type admin_username: str +:param kubernetes_version: Version of Kubernetes specified when creating the managed cluster. +:type kubernetes_version: str +:param node_vm_size: Vm to be spin up. +:type node_vm_size: str or ContainerServiceVMSizeTypes Enum +:param node_osdisk_size: Size in GB to be used to specify the disk size for +every machine in this master/agent pool. If you specify 0, it will apply the default +osDisk size according to the vmSize specified. +:type node_osdisk_size: int +:param node_count: Number of agents (VMs) to host docker containers. +Allowed values must be in the range of 1 to 100 (inclusive). The default value is 1. +:type node_count: int +:param no_ssh_key: Specified if it is linuxprofile. +:type no_ssh_key: boolean +:param vnet_subnet_id: VNet SubnetID specifies the vnet's subnet identifier. +:type vnet_subnet_id: str +:param max_pods: Maximum number of pods that can run on a node. +:type max_pods: int +:param os_type: OsType to be used to specify os type. Choose from Linux and Windows. +Default to Linux. +:type os_type: str or OSType Enum +:param tags: Resource tags. +:type tags: dict[str, str] +:param location: Required resource location +:type location: str + +:Example: + +>>> a = AzureKubernetesOperator(task_id="task",ci_conn_id='azure_kubernetes_default', +resource_group="my_resource_group", +name="my_aks_container", +ssh_key_value=None, +dns_name_prefix=None, +location="my_region", +tags=None +) +""" +@apply_defaults +def __init__(self, ci_conn_id, resource_group, name, ssh_key_value, + dns_name_prefix=None, + location=None, + admin_username="azureuser", + kubernetes_version='', + node_vm_size="Standard_DS2_v2", + node_osdisk_size=0, + node_count=3, +
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r261851617 ## File path: airflow/contrib/operators/aks_operator.py ## @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.contrib.hooks.azure_kubernetes_hook import AzureKubernetesServiceHook +from airflow.models import BaseOperator +from airflow.utils.decorators import apply_defaults + +from azure.mgmt.containerservice.models import ContainerServiceLinuxProfile +from azure.mgmt.containerservice.models import ContainerServiceServicePrincipalProfile +from azure.mgmt.containerservice.models import ContainerServiceSshConfiguration +from azure.mgmt.containerservice.models import ContainerServiceSshPublicKey +from azure.mgmt.containerservice.models import ContainerServiceStorageProfileTypes +from azure.mgmt.containerservice.models import ManagedCluster +from azure.mgmt.containerservice.models import ManagedClusterAgentPoolProfile + +from airflow.contrib.utils.aks_utils import \ +is_valid_ssh_rsa_public_key, get_poller_result, get_public_key, get_default_dns_prefix +from msrestazure.azure_exceptions import CloudError + + +class AzureKubernetesOperator(BaseOperator): +""" +Start a Azure Kubernetes Service + +:param ci_conn_id: connection id of a +service principal which will be used to start the azure kubernetes service +:type ci_conn_id: str +:param resource_group: Required name of the resource group +wherein this container instance should be started +:type resource_group: str +:param name: Required name of this container. Please note this name +has to be unique in order to run containers in parallel. +:type name: str +:param ssh_key_value: the ssh value used to connect to machine to be used +:type ssh_key_value: str +:param dns_name_prefix: DNS prefix specified when creating the managed cluster. +:type region: dns_name_prefix +:param admin_username: The administrator username to use for Linux VMs. +:type admin_username: str +:param kubernetes_version: Version of Kubernetes specified when creating the managed cluster. +:type kubernetes_version: str +:param node_vm_size: Vm to be spin up. +:type node_vm_size: str or ContainerServiceVMSizeTypes Enum +:param node_osdisk_size: Size in GB to be used to specify the disk size for +every machine in this master/agent pool. If you specify 0, it will apply the default +osDisk size according to the vmSize specified. +:type node_osdisk_size: int +:param node_count: Number of agents (VMs) to host docker containers. +Allowed values must be in the range of 1 to 100 (inclusive). The default value is 1. +:type node_count: int +:param no_ssh_key: Specified if it is linuxprofile. +:type no_ssh_key: boolean +:param vnet_subnet_id: VNet SubnetID specifies the vnet's subnet identifier. +:type vnet_subnet_id: str +:param max_pods: Maximum number of pods that can run on a node. +:type max_pods: int +:param os_type: OsType to be used to specify os type. Choose from Linux and Windows. +Default to Linux. +:type os_type: str or OSType Enum +:param tags: Resource tags. +:type tags: dict[str, str] +:param location: Required resource location +:type location: str + +:Example: + +>>> a = AzureKubernetesOperator(task_id="task",ci_conn_id='azure_kubernetes_default', +resource_group="my_resource_group", +name="my_aks_container", +ssh_key_value=None, +dns_name_prefix=None, +location="my_region", +tags=None +) +""" +@apply_defaults +def __init__(self, ci_conn_id, resource_group, name, ssh_key_value, + dns_name_prefix=None, + location=None, + admin_username="azureuser", + kubernetes_version='', + node_vm_size="Standard_DS2_v2", + node_osdisk_size=0, + node_count=3, +
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r261851602 ## File path: airflow/contrib/operators/aks_operator.py ## @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.contrib.hooks.azure_kubernetes_hook import AzureKubernetesServiceHook +from airflow.models import BaseOperator +from airflow.utils.decorators import apply_defaults + +from azure.mgmt.containerservice.models import ContainerServiceLinuxProfile +from azure.mgmt.containerservice.models import ContainerServiceServicePrincipalProfile +from azure.mgmt.containerservice.models import ContainerServiceSshConfiguration +from azure.mgmt.containerservice.models import ContainerServiceSshPublicKey +from azure.mgmt.containerservice.models import ContainerServiceStorageProfileTypes +from azure.mgmt.containerservice.models import ManagedCluster +from azure.mgmt.containerservice.models import ManagedClusterAgentPoolProfile + +from airflow.contrib.utils.aks_utils import \ +is_valid_ssh_rsa_public_key, get_poller_result, get_public_key, get_default_dns_prefix +from msrestazure.azure_exceptions import CloudError + + +class AzureKubernetesOperator(BaseOperator): +""" +Start a Azure Kubernetes Service + +:param ci_conn_id: connection id of a +service principal which will be used to start the azure kubernetes service +:type ci_conn_id: str +:param resource_group: Required name of the resource group +wherein this container instance should be started +:type resource_group: str +:param name: Required name of this container. Please note this name +has to be unique in order to run containers in parallel. +:type name: str +:param ssh_key_value: the ssh value used to connect to machine to be used +:type ssh_key_value: str +:param dns_name_prefix: DNS prefix specified when creating the managed cluster. +:type region: dns_name_prefix +:param admin_username: The administrator username to use for Linux VMs. +:type admin_username: str +:param kubernetes_version: Version of Kubernetes specified when creating the managed cluster. +:type kubernetes_version: str +:param node_vm_size: Vm to be spin up. +:type node_vm_size: str or ContainerServiceVMSizeTypes Enum +:param node_osdisk_size: Size in GB to be used to specify the disk size for +every machine in this master/agent pool. If you specify 0, it will apply the default +osDisk size according to the vmSize specified. +:type node_osdisk_size: int +:param node_count: Number of agents (VMs) to host docker containers. +Allowed values must be in the range of 1 to 100 (inclusive). The default value is 1. +:type node_count: int +:param no_ssh_key: Specified if it is linuxprofile. +:type no_ssh_key: boolean +:param vnet_subnet_id: VNet SubnetID specifies the vnet's subnet identifier. +:type vnet_subnet_id: str +:param max_pods: Maximum number of pods that can run on a node. +:type max_pods: int +:param os_type: OsType to be used to specify os type. Choose from Linux and Windows. +Default to Linux. +:type os_type: str or OSType Enum +:param tags: Resource tags. +:type tags: dict[str, str] +:param location: Required resource location +:type location: str + +:Example: + +>>> a = AzureKubernetesOperator(task_id="task",ci_conn_id='azure_kubernetes_default', +resource_group="my_resource_group", +name="my_aks_container", +ssh_key_value=None, +dns_name_prefix=None, +location="my_region", +tags=None +) +""" +@apply_defaults +def __init__(self, ci_conn_id, resource_group, name, ssh_key_value, + dns_name_prefix=None, + location=None, + admin_username="azureuser", + kubernetes_version='', + node_vm_size="Standard_DS2_v2", + node_osdisk_size=0, + node_count=3, +
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement
Azure Kubernetes Service Operator
URL: https://github.com/apache/airflow/pull/4530#discussion_r256590452
##
File path: airflow/utils/db.py
##
@@ -284,6 +284,10 @@ def initdb():
Connection(
conn_id='azure_container_instances_default',
conn_type='azure_container_instances',
extra='{"tenantId": "", "subscriptionId": "" }'))
+merge_conn(
Review comment:
I think you should create one connection for all Azure services ex.
`azure_default`. Look at:
https://github.com/apache/airflow/pull/4494/files#r249418220
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r256589015 ## File path: docs/integration.rst ## @@ -271,6 +271,23 @@ AzureContainerVolumeHook .. _AWS: +Azure Kubernetes Service +' +Azure Kubernetes Service (AKS) simplifies the deployment and operations of Kubernetes and enables you to dynamically scale your application infrastructure. +The AzureKubernetesServiceHook requires a service principal. The credentials for this principal can either be defined in the extra field `key_path`, as an +environment variable named `AZURE_AUTH_LOCATION`. + +- :ref:`AzureKubernetesOperator` : Start new AKS. +- :ref:`AzureKubernetesServiceHook` : Wrapper around a aks. + +AzureKubernetesOperator +""" + .. autoclass:: airflow.contrib.operators.aks_operator.AzureKubernetesOperator + Review comment: Now it does not matter anymore. The structure of the integration.rst file has changed. Look how this file looks like now if you have an autoclass directive in the file, you can refer to it by reference: class:. If you have an autoclass directive in two files then you need to decide which is more important and to which links should be addressed. For this purpose, you add a noindex flag to lower-order directives. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r256589015 ## File path: docs/integration.rst ## @@ -271,6 +271,23 @@ AzureContainerVolumeHook .. _AWS: +Azure Kubernetes Service +' +Azure Kubernetes Service (AKS) simplifies the deployment and operations of Kubernetes and enables you to dynamically scale your application infrastructure. +The AzureKubernetesServiceHook requires a service principal. The credentials for this principal can either be defined in the extra field `key_path`, as an +environment variable named `AZURE_AUTH_LOCATION`. + +- :ref:`AzureKubernetesOperator` : Start new AKS. +- :ref:`AzureKubernetesServiceHook` : Wrapper around a aks. + +AzureKubernetesOperator +""" + .. autoclass:: airflow.contrib.operators.aks_operator.AzureKubernetesOperator + Review comment: Now it does not matter anymore. The structure of the integration.rst file has changed. Look how this file looks like now if you have an autoclass directive in the file, you can refer to it by reference: class:. If you have an autoclass directive in two files then you need to decide which is more important and to which links should be addressed. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r256586886 ## File path: airflow/contrib/hooks/azure_kubernetes_hook.py ## @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +from airflow.hooks.base_hook import BaseHook +from airflow.exceptions import AirflowException + +from azure.common.credentials import ServicePrincipalCredentials +from azure.mgmt.containerservice import ContainerServiceClient +from azure.mgmt.resource import ResourceManagementClient +from airflow.contrib.utils.aks_utils import load_json + + +class AzureKubernetesServiceHook(BaseHook): + +def __init__(self, conn_id=None): Review comment: Set the default value for conn_id. This is required for the connection mechanism built into Airflow. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement
Azure Kubernetes Service Operator
URL: https://github.com/apache/airflow/pull/4530#discussion_r256586212
##
File path: airflow/contrib/hooks/azure_kubernetes_hook.py
##
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+
+from airflow.hooks.base_hook import BaseHook
+from airflow.exceptions import AirflowException
+
+from azure.common.credentials import ServicePrincipalCredentials
+from azure.mgmt.containerservice import ContainerServiceClient
+from azure.mgmt.resource import ResourceManagementClient
+from airflow.contrib.utils.aks_utils import load_json
+
+
+class AzureKubernetesServiceHook(BaseHook):
+
+def __init__(self, conn_id=None):
+self.conn_id = conn_id
+self.connection = self.get_conn()
+self.configData = None
+self.credentials = None
+self.subscription_id = None
+self.clientId = None
+self.clientSecret = None
+
+def get_conn(self):
+if self.conn_id:
+conn = self.get_connection(self.conn_id)
+key_path = conn.extra_dejson.get('key_path', False)
+if key_path:
+if key_path.endswith('.json'):
+self.log.info('Getting connection using a JSON key file.')
+
+self.configData = load_json(self, key_path)
+else:
+raise AirflowException('Unrecognised extension for key
file.')
+
+if os.environ.get('AZURE_AUTH_LOCATION'):
Review comment:
@apraovjr I do not know what more I can say except what I wrote earlier.
Airflow supports the definition of connections via environment variables. The
Airfllow solution is better than the one presented in this code, because
Airflow supports the definition of multiple connections using environment
variables.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r255242729 ## File path: setup.py ## @@ -337,6 +342,7 @@ def do_setup(): 'async': async_packages, 'azure_blob_storage': azure_blob_storage, 'azure_data_lake': azure_data_lake, +'azure_kubernetes_service': azure_kubernetes_service, Review comment: I submit PR a few days ago. It was merged today. Look at: https://github.com/apache/airflow/pull/4524 This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r254998869 ## File path: docs/integration.rst ## @@ -271,6 +271,23 @@ AzureContainerVolumeHook .. _AWS: +Azure Kubernetes Service +' +Azure Kubernetes Service (AKS) simplifies the deployment and operations of Kubernetes and enables you to dynamically scale your application infrastructure. +The AzureKubernetesServiceHook requires a service principal. The credentials for this principal can either be defined in the extra field `key_path`, as an +environment variable named `AZURE_AUTH_LOCATION`. + +- :ref:`AzureKubernetesOperator` : Start new AKS. +- :ref:`AzureKubernetesServiceHook` : Wrapper around a aks. + +AzureKubernetesOperator +""" + .. autoclass:: airflow.contrib.operators.aks_operator.AzureKubernetesOperator + +AzureKubernetesServiceHook +"" + .. autoclass:: airflow.contrib.hooks.azure_kubernetes_hook.AzureKubernetesServiceHook Review comment: The structure of `integration.rst` file has changed. Can you update your PRs? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r249205505 ## File path: setup.py ## @@ -337,6 +342,7 @@ def do_setup(): 'async': async_packages, 'azure_blob_storage': azure_blob_storage, 'azure_data_lake': azure_data_lake, +'azure_kubernetes_service': azure_kubernetes_service, Review comment: Do you really need a new group? Google Cloud Platform (`gcp_api`) does not create a separate group for each of services This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r253279418 ## File path: docs/integration.rst ## @@ -271,6 +271,23 @@ AzureContainerVolumeHook .. _AWS: +Azure Kubernetes Service +' +Azure Kubernetes Service (AKS) simplifies the deployment and operations of Kubernetes and enables you to dynamically scale your application infrastructure. +The AzureKubernetesServiceHook requires a service principal. The credentials for this principal can either be defined in the extra field `key_path`, as an +environment variable named `AZURE_AUTH_LOCATION`. + +- :ref:`AzureKubernetesOperator` : Start new AKS. +- :ref:`AzureKubernetesServiceHook` : Wrapper around a aks. + +AzureKubernetesOperator +""" + .. autoclass:: airflow.contrib.operators.aks_operator.AzureKubernetesOperator + +AzureKubernetesServiceHook +"" + .. autoclass:: airflow.contrib.hooks.azure_kubernetes_hook.AzureKubernetesServiceHook + Review comment: Ditto. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r249205505 ## File path: setup.py ## @@ -337,6 +342,7 @@ def do_setup(): 'async': async_packages, 'azure_blob_storage': azure_blob_storage, 'azure_data_lake': azure_data_lake, +'azure_kubernetes_service': azure_kubernetes_service, Review comment: Do you really need a new group? Google Cloud Platform (`gcp_api`) does not create a separate group for each of its services This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r253279416 ## File path: docs/integration.rst ## @@ -271,6 +271,23 @@ AzureContainerVolumeHook .. _AWS: +Azure Kubernetes Service +' +Azure Kubernetes Service (AKS) simplifies the deployment and operations of Kubernetes and enables you to dynamically scale your application infrastructure. +The AzureKubernetesServiceHook requires a service principal. The credentials for this principal can either be defined in the extra field `key_path`, as an +environment variable named `AZURE_AUTH_LOCATION`. + +- :ref:`AzureKubernetesOperator` : Start new AKS. +- :ref:`AzureKubernetesServiceHook` : Wrapper around a aks. + +AzureKubernetesOperator +""" + .. autoclass:: airflow.contrib.operators.aks_operator.AzureKubernetesOperator + Review comment: Missing `no-index` flag. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement
Azure Kubernetes Service Operator
URL: https://github.com/apache/airflow/pull/4530#discussion_r249239186
##
File path: airflow/contrib/utils/aks_utils.py
##
@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+import json
+import re
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.primitives import serialization
+
+from knack.log import get_logger
+logger = get_logger(__name__)
+
+
+def is_valid_ssh_rsa_public_key(openssh_pubkey):
+#
http://stackoverflow.com/questions/2494450/ssh-rsa-public-key-validation-using-a-regular-expression
+# A "good enough" check is to see if the key starts with the correct
header.
+import struct
+try:
+from base64 import decodebytes as base64_decode
+except ImportError:
+# deprecated and redirected to decodebytes in Python 3
+from base64 import decodestring as base64_decode
+
+parts = openssh_pubkey.split()
+if len(parts) < 2:
+return False
+key_type = parts[0]
+key_string = parts[1]
+
+data = base64_decode(key_string.encode()) #
pylint:disable=deprecated-method
+int_len = 4
+str_len = struct.unpack('>I', data[:int_len])[0] # this should return 7
+return data[int_len:int_len + str_len] == key_type.encode()
+
+
+def get_public_key(self):
+key = rsa.generate_private_key(backend=default_backend(),
public_exponent=65537, key_size=2048)
+return key.public_key().public_bytes(serialization.Encoding.OpenSSH,
+
serialization.PublicFormat.OpenSSH).decode('utf-8')
+
+
+def load_json(self, file_path):
+try:
+with open(file_path) as configFile:
+configData = json.load(configFile)
+except FileNotFoundError:
+print("Error: Expecting azurermconfig.json in current folder")
+sys.exit()
Review comment:
This is not a good solution. Probably, when an error occurs, the program
will be closed without reporting an error. If an error has occurred then you
should throw an exception.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r249205052 ## File path: airflow/contrib/operators/aks_operator.py ## @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.contrib.hooks.azure_kubernetes_hook import AzureKubernetesServiceHook +from airflow.models import BaseOperator + +from azure.mgmt.containerservice.models import ContainerServiceLinuxProfile +from azure.mgmt.containerservice.models import ContainerServiceServicePrincipalProfile +from azure.mgmt.containerservice.models import ContainerServiceSshConfiguration +from azure.mgmt.containerservice.models import ContainerServiceSshPublicKey +from azure.mgmt.containerservice.models import ContainerServiceStorageProfileTypes +from azure.mgmt.containerservice.models import ManagedCluster +from azure.mgmt.containerservice.models import ManagedClusterAgentPoolProfile + +from airflow.contrib.utils.aks_utils import \ +is_valid_ssh_rsa_public_key, get_poller_result, get_public_key, get_default_dns_prefix +from knack.log import get_logger +from msrestazure.azure_exceptions import CloudError + +logger = get_logger(__name__) + + +class AzureKubernetesOperator(BaseOperator): +""" +Start a Azure Kubernetes Service + +:param ci_conn_id: connection id of a +service principal which will be used to start the azure kubernetes service +:type ci_conn_id: str +:param resource_group: Required name of the resource group +wherein this container instance should be started +:type resource_group: str +:param name: Required name of this container. Please note this name +has to be unique in order to run containers in parallel. +:type name: str +:param ssh_key_value: the ssh value used to connect to machine to be used +:type ssh_key_value: str +:param dns_name_prefix: DNS prefix specified when creating the managed cluster. +:type region: dns_name_prefix +:param admin_username: The administrator username to use for Linux VMs. +:type admin_username: str +:param kubernetes_version: Version of Kubernetes specified when creating the managed cluster. +:type kubernetes_version: str +:param node_vm_size: Vm to be spin up. +:type node_vm_size: str or ContainerServiceVMSizeTypes Enum +:param node_osdisk_size: Size in GB to be used to specify the disk size for +every machine in this master/agent pool. If you specify 0, it will apply the default +osDisk size according to the vmSize specified. +:type node_osdisk_size: int +:param node_count: Number of agents (VMs) to host docker containers. +Allowed values must be in the range of 1 to 100 (inclusive). The default value is 1. +:type node_count: int +:param no_ssh_key: Specified if it is linuxprofile. +:type no_ssh_key: boolean +:param vnet_subnet_id: VNet SubnetID specifies the vnet's subnet identifier. +:type vnet_subnet_id: str +:param max_pods: Maximum number of pods that can run on a node. +:type max_pods: int +:param os_type: OsType to be used to specify os type. Choose from Linux and Windows. +Default to Linux. +:type os_type: str or OSType Enum +:param tags: Resource tags. +:type tags: dict[str, str] +:param location: Required resource location +:type location: str + +:Example: + +>>> a = AzureKubernetesOperator(task_id="task",ci_conn_id='azure_kubernetes_default', +resource_group="my_resource_group", +name="my_aks_container", +ssh_key_value=None, +dns_name_prefix=None, +location="my_region", +tags=None +) +""" + +def __init__(self, ci_conn_id, resource_group, name, ssh_key_value, + dns_name_prefix=None, + location=None, + admin_username="azureuser", + kubernetes_version='', + node_vm_size="Standard_DS2_v2", + node_osdisk_size=0, + node_count=3, + no_ssh_key=False, +
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r249205052 ## File path: airflow/contrib/operators/aks_operator.py ## @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.contrib.hooks.azure_kubernetes_hook import AzureKubernetesServiceHook +from airflow.models import BaseOperator + +from azure.mgmt.containerservice.models import ContainerServiceLinuxProfile +from azure.mgmt.containerservice.models import ContainerServiceServicePrincipalProfile +from azure.mgmt.containerservice.models import ContainerServiceSshConfiguration +from azure.mgmt.containerservice.models import ContainerServiceSshPublicKey +from azure.mgmt.containerservice.models import ContainerServiceStorageProfileTypes +from azure.mgmt.containerservice.models import ManagedCluster +from azure.mgmt.containerservice.models import ManagedClusterAgentPoolProfile + +from airflow.contrib.utils.aks_utils import \ +is_valid_ssh_rsa_public_key, get_poller_result, get_public_key, get_default_dns_prefix +from knack.log import get_logger +from msrestazure.azure_exceptions import CloudError + +logger = get_logger(__name__) + + +class AzureKubernetesOperator(BaseOperator): +""" +Start a Azure Kubernetes Service + +:param ci_conn_id: connection id of a +service principal which will be used to start the azure kubernetes service +:type ci_conn_id: str +:param resource_group: Required name of the resource group +wherein this container instance should be started +:type resource_group: str +:param name: Required name of this container. Please note this name +has to be unique in order to run containers in parallel. +:type name: str +:param ssh_key_value: the ssh value used to connect to machine to be used +:type ssh_key_value: str +:param dns_name_prefix: DNS prefix specified when creating the managed cluster. +:type region: dns_name_prefix +:param admin_username: The administrator username to use for Linux VMs. +:type admin_username: str +:param kubernetes_version: Version of Kubernetes specified when creating the managed cluster. +:type kubernetes_version: str +:param node_vm_size: Vm to be spin up. +:type node_vm_size: str or ContainerServiceVMSizeTypes Enum +:param node_osdisk_size: Size in GB to be used to specify the disk size for +every machine in this master/agent pool. If you specify 0, it will apply the default +osDisk size according to the vmSize specified. +:type node_osdisk_size: int +:param node_count: Number of agents (VMs) to host docker containers. +Allowed values must be in the range of 1 to 100 (inclusive). The default value is 1. +:type node_count: int +:param no_ssh_key: Specified if it is linuxprofile. +:type no_ssh_key: boolean +:param vnet_subnet_id: VNet SubnetID specifies the vnet's subnet identifier. +:type vnet_subnet_id: str +:param max_pods: Maximum number of pods that can run on a node. +:type max_pods: int +:param os_type: OsType to be used to specify os type. Choose from Linux and Windows. +Default to Linux. +:type os_type: str or OSType Enum +:param tags: Resource tags. +:type tags: dict[str, str] +:param location: Required resource location +:type location: str + +:Example: + +>>> a = AzureKubernetesOperator(task_id="task",ci_conn_id='azure_kubernetes_default', +resource_group="my_resource_group", +name="my_aks_container", +ssh_key_value=None, +dns_name_prefix=None, +location="my_region", +tags=None +) +""" + +def __init__(self, ci_conn_id, resource_group, name, ssh_key_value, + dns_name_prefix=None, + location=None, + admin_username="azureuser", + kubernetes_version='', + node_vm_size="Standard_DS2_v2", + node_osdisk_size=0, + node_count=3, + no_ssh_key=False, +
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r249203936 ## File path: airflow/contrib/operators/aks_operator.py ## @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.contrib.hooks.azure_kubernetes_hook import AzureKubernetesServiceHook +from airflow.models import BaseOperator + +from azure.mgmt.containerservice.models import ContainerServiceLinuxProfile +from azure.mgmt.containerservice.models import ContainerServiceServicePrincipalProfile +from azure.mgmt.containerservice.models import ContainerServiceSshConfiguration +from azure.mgmt.containerservice.models import ContainerServiceSshPublicKey +from azure.mgmt.containerservice.models import ContainerServiceStorageProfileTypes +from azure.mgmt.containerservice.models import ManagedCluster +from azure.mgmt.containerservice.models import ManagedClusterAgentPoolProfile + +from airflow.contrib.utils.aks_utils import \ +is_valid_ssh_rsa_public_key, get_poller_result, get_public_key, get_default_dns_prefix +from knack.log import get_logger +from msrestazure.azure_exceptions import CloudError + +logger = get_logger(__name__) Review comment: Please do not use a separate logger. `BaseOperator` extends `LoggingMixin`, which has methods for this purpose. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement
Azure Kubernetes Service Operator
URL: https://github.com/apache/airflow/pull/4530#discussion_r249239029
##
File path: airflow/contrib/hooks/azure_kubernetes_hook.py
##
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+
+from airflow.hooks.base_hook import BaseHook
+from airflow.exceptions import AirflowException
+
+from azure.common.credentials import ServicePrincipalCredentials
+from azure.mgmt.containerservice import ContainerServiceClient
+from azure.mgmt.resource import ResourceManagementClient
+from airflow.contrib.utils.aks_utils import load_json
+
+
+class AzureKubernetesServiceHook(BaseHook):
+
+def __init__(self, conn_id=None):
+self.conn_id = conn_id
+self.connection = self.get_conn()
+self.configData = None
+self.credentials = None
+self.subscription_id = None
+self.clientId = None
+self.clientSecret = None
+
+def get_conn(self):
+if self.conn_id:
+conn = self.get_connection(self.conn_id)
+key_path = conn.extra_dejson.get('key_path', False)
+if key_path:
+if key_path.endswith('.json'):
+self.log.info('Getting connection using a JSON key file.')
+
+self.configData = load_json(self, key_path)
+else:
+raise AirflowException('Unrecognised extension for key
file.')
+
+if os.environ.get('AZURE_AUTH_LOCATION'):
Review comment:
I found similar looking code in another place:
https://github.com/apache/airflow/blob/master/airflow/contrib/hooks/azure_container_instance_hook.py
The user can define a new connection using the environment variable
`AIRFLOW_CONN_*`. For example: `AIRFLOW_CONN_AZURE_DEFAULT`. Official
documentation is missing, but you can look at this PR:
https://github.com/apache/airflow/pull/4523/files
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r249203936 ## File path: airflow/contrib/operators/aks_operator.py ## @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.contrib.hooks.azure_kubernetes_hook import AzureKubernetesServiceHook +from airflow.models import BaseOperator + +from azure.mgmt.containerservice.models import ContainerServiceLinuxProfile +from azure.mgmt.containerservice.models import ContainerServiceServicePrincipalProfile +from azure.mgmt.containerservice.models import ContainerServiceSshConfiguration +from azure.mgmt.containerservice.models import ContainerServiceSshPublicKey +from azure.mgmt.containerservice.models import ContainerServiceStorageProfileTypes +from azure.mgmt.containerservice.models import ManagedCluster +from azure.mgmt.containerservice.models import ManagedClusterAgentPoolProfile + +from airflow.contrib.utils.aks_utils import \ +is_valid_ssh_rsa_public_key, get_poller_result, get_public_key, get_default_dns_prefix +from knack.log import get_logger +from msrestazure.azure_exceptions import CloudError + +logger = get_logger(__name__) Review comment: Please do not use a separate logger. `BaseOperator extends `LoggingMixin`, which has methods for this purpose. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r249205505 ## File path: setup.py ## @@ -337,6 +342,7 @@ def do_setup(): 'async': async_packages, 'azure_blob_storage': azure_blob_storage, 'azure_data_lake': azure_data_lake, +'azure_kubernetes_service': azure_kubernetes_service, Review comment: Do you really need a new group? Google Cloud Platform (`gcp_api`) does not create a separate group for each of your services This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r249205052 ## File path: airflow/contrib/operators/aks_operator.py ## @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.contrib.hooks.azure_kubernetes_hook import AzureKubernetesServiceHook +from airflow.models import BaseOperator + +from azure.mgmt.containerservice.models import ContainerServiceLinuxProfile +from azure.mgmt.containerservice.models import ContainerServiceServicePrincipalProfile +from azure.mgmt.containerservice.models import ContainerServiceSshConfiguration +from azure.mgmt.containerservice.models import ContainerServiceSshPublicKey +from azure.mgmt.containerservice.models import ContainerServiceStorageProfileTypes +from azure.mgmt.containerservice.models import ManagedCluster +from azure.mgmt.containerservice.models import ManagedClusterAgentPoolProfile + +from airflow.contrib.utils.aks_utils import \ +is_valid_ssh_rsa_public_key, get_poller_result, get_public_key, get_default_dns_prefix +from knack.log import get_logger +from msrestazure.azure_exceptions import CloudError + +logger = get_logger(__name__) + + +class AzureKubernetesOperator(BaseOperator): +""" +Start a Azure Kubernetes Service + +:param ci_conn_id: connection id of a +service principal which will be used to start the azure kubernetes service +:type ci_conn_id: str +:param resource_group: Required name of the resource group +wherein this container instance should be started +:type resource_group: str +:param name: Required name of this container. Please note this name +has to be unique in order to run containers in parallel. +:type name: str +:param ssh_key_value: the ssh value used to connect to machine to be used +:type ssh_key_value: str +:param dns_name_prefix: DNS prefix specified when creating the managed cluster. +:type region: dns_name_prefix +:param admin_username: The administrator username to use for Linux VMs. +:type admin_username: str +:param kubernetes_version: Version of Kubernetes specified when creating the managed cluster. +:type kubernetes_version: str +:param node_vm_size: Vm to be spin up. +:type node_vm_size: str or ContainerServiceVMSizeTypes Enum +:param node_osdisk_size: Size in GB to be used to specify the disk size for +every machine in this master/agent pool. If you specify 0, it will apply the default +osDisk size according to the vmSize specified. +:type node_osdisk_size: int +:param node_count: Number of agents (VMs) to host docker containers. +Allowed values must be in the range of 1 to 100 (inclusive). The default value is 1. +:type node_count: int +:param no_ssh_key: Specified if it is linuxprofile. +:type no_ssh_key: boolean +:param vnet_subnet_id: VNet SubnetID specifies the vnet's subnet identifier. +:type vnet_subnet_id: str +:param max_pods: Maximum number of pods that can run on a node. +:type max_pods: int +:param os_type: OsType to be used to specify os type. Choose from Linux and Windows. +Default to Linux. +:type os_type: str or OSType Enum +:param tags: Resource tags. +:type tags: dict[str, str] +:param location: Required resource location +:type location: str + +:Example: + +>>> a = AzureKubernetesOperator(task_id="task",ci_conn_id='azure_kubernetes_default', +resource_group="my_resource_group", +name="my_aks_container", +ssh_key_value=None, +dns_name_prefix=None, +location="my_region", +tags=None +) +""" + +def __init__(self, ci_conn_id, resource_group, name, ssh_key_value, + dns_name_prefix=None, + location=None, + admin_username="azureuser", + kubernetes_version='', + node_vm_size="Standard_DS2_v2", + node_osdisk_size=0, + node_count=3, + no_ssh_key=False, +
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r249204162 ## File path: airflow/contrib/operators/aks_operator.py ## @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.contrib.hooks.azure_kubernetes_hook import AzureKubernetesServiceHook +from airflow.models import BaseOperator + +from azure.mgmt.containerservice.models import ContainerServiceLinuxProfile +from azure.mgmt.containerservice.models import ContainerServiceServicePrincipalProfile +from azure.mgmt.containerservice.models import ContainerServiceSshConfiguration +from azure.mgmt.containerservice.models import ContainerServiceSshPublicKey +from azure.mgmt.containerservice.models import ContainerServiceStorageProfileTypes +from azure.mgmt.containerservice.models import ManagedCluster +from azure.mgmt.containerservice.models import ManagedClusterAgentPoolProfile + +from airflow.contrib.utils.aks_utils import \ +is_valid_ssh_rsa_public_key, get_poller_result, get_public_key, get_default_dns_prefix +from knack.log import get_logger +from msrestazure.azure_exceptions import CloudError + +logger = get_logger(__name__) + + +class AzureKubernetesOperator(BaseOperator): +""" +Start a Azure Kubernetes Service + +:param ci_conn_id: connection id of a +service principal which will be used to start the azure kubernetes service +:type ci_conn_id: str +:param resource_group: Required name of the resource group +wherein this container instance should be started +:type resource_group: str +:param name: Required name of this container. Please note this name +has to be unique in order to run containers in parallel. +:type name: str +:param ssh_key_value: the ssh value used to connect to machine to be used +:type ssh_key_value: str +:param dns_name_prefix: DNS prefix specified when creating the managed cluster. +:type region: dns_name_prefix +:param admin_username: The administrator username to use for Linux VMs. +:type admin_username: str +:param kubernetes_version: Version of Kubernetes specified when creating the managed cluster. +:type kubernetes_version: str +:param node_vm_size: Vm to be spin up. +:type node_vm_size: str or ContainerServiceVMSizeTypes Enum +:param node_osdisk_size: Size in GB to be used to specify the disk size for +every machine in this master/agent pool. If you specify 0, it will apply the default +osDisk size according to the vmSize specified. +:type node_osdisk_size: int +:param node_count: Number of agents (VMs) to host docker containers. +Allowed values must be in the range of 1 to 100 (inclusive). The default value is 1. +:type node_count: int +:param no_ssh_key: Specified if it is linuxprofile. +:type no_ssh_key: boolean +:param vnet_subnet_id: VNet SubnetID specifies the vnet's subnet identifier. +:type vnet_subnet_id: str +:param max_pods: Maximum number of pods that can run on a node. +:type max_pods: int +:param os_type: OsType to be used to specify os type. Choose from Linux and Windows. +Default to Linux. +:type os_type: str or OSType Enum +:param tags: Resource tags. +:type tags: dict[str, str] +:param location: Required resource location +:type location: str + +:Example: + +>>> a = AzureKubernetesOperator(task_id="task",ci_conn_id='azure_kubernetes_default', +resource_group="my_resource_group", +name="my_aks_container", +ssh_key_value=None, +dns_name_prefix=None, +location="my_region", +tags=None +) +""" + +def __init__(self, ci_conn_id, resource_group, name, ssh_key_value, + dns_name_prefix=None, + location=None, + admin_username="azureuser", + kubernetes_version='', + node_vm_size="Standard_DS2_v2", + node_osdisk_size=0, + node_count=3, + no_ssh_key=False, +
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r249203936 ## File path: airflow/contrib/operators/aks_operator.py ## @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.contrib.hooks.azure_kubernetes_hook import AzureKubernetesServiceHook +from airflow.models import BaseOperator + +from azure.mgmt.containerservice.models import ContainerServiceLinuxProfile +from azure.mgmt.containerservice.models import ContainerServiceServicePrincipalProfile +from azure.mgmt.containerservice.models import ContainerServiceSshConfiguration +from azure.mgmt.containerservice.models import ContainerServiceSshPublicKey +from azure.mgmt.containerservice.models import ContainerServiceStorageProfileTypes +from azure.mgmt.containerservice.models import ManagedCluster +from azure.mgmt.containerservice.models import ManagedClusterAgentPoolProfile + +from airflow.contrib.utils.aks_utils import \ +is_valid_ssh_rsa_public_key, get_poller_result, get_public_key, get_default_dns_prefix +from knack.log import get_logger +from msrestazure.azure_exceptions import CloudError + +logger = get_logger(__name__) Review comment: Please do not use a separate logger. `BaseOperator` will extends `LoggingMixin`, which has methods for this purpose. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement
Azure Kubernetes Service Operator
URL: https://github.com/apache/airflow/pull/4530#discussion_r247741171
##
File path: airflow/models/connection.py
##
@@ -84,6 +84,7 @@ class Connection(Base, LoggingMixin):
('segment', 'Segment',),
('azure_data_lake', 'Azure Data Lake'),
('azure_container_instances', 'Azure Container Instances'),
+('azure_kubernetes_instances', 'Azure Kubernetes Instances'),
Review comment:
@apraovjr However, you would like to know that we are very grateful for this
PR, but I think that we should reflect on certain matters before this PR can be
accepted. In the last 3 days, this is second PR related to Azure that has
similar comments.
https://github.com/apache/airflow/pulls?utf8=%E2%9C%93=is%3Apr+is%3Aopen+Azure
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement
Azure Kubernetes Service Operator
URL: https://github.com/apache/airflow/pull/4530#discussion_r247739678
##
File path: airflow/models/connection.py
##
@@ -84,6 +84,7 @@ class Connection(Base, LoggingMixin):
('segment', 'Segment',),
('azure_data_lake', 'Azure Data Lake'),
('azure_container_instances', 'Azure Container Instances'),
+('azure_kubernetes_instances', 'Azure Kubernetes Instances'),
Review comment:
Is this required? Google Cloud Platform has only one hook. GCP SQL has a
separate, but this is a special case.
I am calling @asbh, because we deal with "creativity" of Azura in [another
PR](https://github.com/apache/airflow/pull/4524), and this change is loosely
connected. We probably have to make systemic decisions on how to limit the
creativity of Azure developer.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement
Azure Kubernetes Service Operator
URL: https://github.com/apache/airflow/pull/4530#discussion_r247739678
##
File path: airflow/models/connection.py
##
@@ -84,6 +84,7 @@ class Connection(Base, LoggingMixin):
('segment', 'Segment',),
('azure_data_lake', 'Azure Data Lake'),
('azure_container_instances', 'Azure Container Instances'),
+('azure_kubernetes_instances', 'Azure Kubernetes Instances'),
Review comment:
Is this required? Google Cloud Platform has only one hook. GCP SQL has a
separate, but this is a special case.
I am calling @asbh, because we deal with "creativity" of Azura in (another
PR)[https://github.com/apache/airflow/pull/4524], and this change is loosely
connected. We probably have to make systemic decisions on how to limit the
creativity of Azure programmers.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r247738173 ## File path: airflow/contrib/operators/aks_operator.py ## @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.contrib.hooks.azure_kubernetes_hook import AzureKubernetesServiceHook +from airflow.models import BaseOperator + +from azure.mgmt.containerservice.models import ContainerServiceLinuxProfile +from azure.mgmt.containerservice.models import ContainerServiceServicePrincipalProfile +from azure.mgmt.containerservice.models import ContainerServiceSshConfiguration +from azure.mgmt.containerservice.models import ContainerServiceSshPublicKey +from azure.mgmt.containerservice.models import ContainerServiceStorageProfileTypes +from azure.mgmt.containerservice.models import ManagedCluster +from azure.mgmt.containerservice.models import ManagedClusterAgentPoolProfile + +from airflow.contrib.utils.aks_utils import \ +is_valid_ssh_rsa_public_key, get_poller_result, get_public_key, get_default_dns_prefix +from knack.log import get_logger +from msrestazure.azure_exceptions import CloudError + +logger = get_logger(__name__) + + +class AzureKubernetesOperator(BaseOperator): +""" +Start a Azure Kubernetes Service + +:param ci_conn_id: connection id of a +service principal which will be used to start the azure kubernetes service +:type ci_conn_id: str +:param resource_group: Required name of the resource group +wherein this container instance should be started +:type resource_group: str +:param name: Required name of this container. Please note this name +has to be unique in order to run containers in parallel. +:type name: str +:param ssh_key_value: the ssh value used to connect to machine to be used +:type ssh_key_value: str +:param dns_name_prefix: DNS prefix specified when creating the managed cluster. +:type region: dns_name_prefix +:param admin_username: The administrator username to use for Linux VMs. +:type admin_username: str +:param kubernetes_version: Version of Kubernetes specified when creating the managed cluster. +:type kubernetes_version: str +:param node_vm_size: Vm to be spin up. +:type node_vm_size: str or ContainerServiceVMSizeTypes Enum +:param node_osdisk_size: Size in GB to be used to specify the disk size for +every machine in this master/agent pool. If you specify 0, it will apply the default +osDisk size according to the vmSize specified. +:type node_osdisk_size: int +:param node_count: Number of agents (VMs) to host docker containers. +Allowed values must be in the range of 1 to 100 (inclusive). The default value is 1. +:type node_count: int +:param no_ssh_key: Specified if it is linuxprofile. +:type no_ssh_key: boolean +:param vnet_subnet_id: VNet SubnetID specifies the vnet's subnet identifier. +:type vnet_subnet_id: str +:param max_pods: Maximum number of pods that can run on a node. +:type max_pods: int +:param os_type: OsType to be used to specify os type. Choose from Linux and Windows. +Default to Linux. +:type os_type: str or OSType Enum +:param tags: Resource tags. +:type tags: dict[str, str] +:param location: Required resource location +:type location: str + +:Example: + +>>> a = AzureKubernetesOperator(task_id="task",ci_conn_id='azure_kubernetes_default', +resource_group="my_resource_group", +name="my_aks_container", +ssh_key_value=None, +dns_name_prefix=None, +location="my_region", +tags=None +) +""" + +def __init__(self, ci_conn_id, resource_group, name, ssh_key_value, Review comment: Please add `@apply_defaults`. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please
[GitHub] mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator
mik-laj commented on a change in pull request #4530: [AIRFLOW-3282] Implement Azure Kubernetes Service Operator URL: https://github.com/apache/airflow/pull/4530#discussion_r247738075 ## File path: airflow/contrib/operators/aks_operator.py ## @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.contrib.hooks.azure_kubernetes_hook import AzureKubernetesServiceHook +from airflow.models import BaseOperator + +from azure.mgmt.containerservice.models import ContainerServiceLinuxProfile +from azure.mgmt.containerservice.models import ContainerServiceServicePrincipalProfile +from azure.mgmt.containerservice.models import ContainerServiceSshConfiguration +from azure.mgmt.containerservice.models import ContainerServiceSshPublicKey +from azure.mgmt.containerservice.models import ContainerServiceStorageProfileTypes +from azure.mgmt.containerservice.models import ManagedCluster +from azure.mgmt.containerservice.models import ManagedClusterAgentPoolProfile + +from airflow.contrib.utils.aks_utils import \ +is_valid_ssh_rsa_public_key, get_poller_result, get_public_key, get_default_dns_prefix +from knack.log import get_logger +from msrestazure.azure_exceptions import CloudError + +logger = get_logger(__name__) + + +class AzureKubernetesOperator(BaseOperator): Review comment: Please add `@apply_defaults`. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
