[
https://issues.apache.org/jira/browse/AIRFLOW-6621?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Mike Prior updated AIRFLOW-6621:
--------------------------------
Description:
Hi
In 10.6, the podoperator works with the code below. Airflow is running in a
docker container, with a valid kubernetes config file. In 1.10.7 the code fails
with various exceptions relating to reading the config.
------------------------------------------------------------------------------
from airflow import DAG
from datetime import datetime, timedelta
from airflow.contrib.operators.kubernetes_pod_operator import
KubernetesPodOperator
from airflow.operators.dummy_operator import DummyOperator
default_args =
{ 'owner': 'mikep', 'depends_on_past': False, 'start_date': datetime(2020, 1,
22), 'retries': 0, 'retry_delay': timedelta(minutes=1) }
dag = DAG('podtest1',
default_args=default_args,
catchup=False,
schedule_interval=None)
doPod = KubernetesPodOperator(
namespace="mynamespace",
image="mypythonimage",
is_delete_operator_pod=True,
cmds=["python","-c"],
arguments=["print('hello world')"],
name="mike-pod-operator",
task_id="pod-operator",
config_file="/app/config",
get_logs=True,
dag=dag
---------------------------------------------------------------
[2020-01-23 07:26:23,245] \{logging_mixin.py:112} INFO - Running %s on host %s
<TaskInstance: podtest1.pod-operator 2020-01-23T13:26:16.911219+00:00
[running]> mypc
[2020-01-23 07:26:23,266] \{taskinstance.py:1088} ERROR - Service host/port is
not set.
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/airflow/models/taskinstance.py",
line 955, in _run_raw_task
result = task_copy.execute(context=context)
File
"/usr/local/lib/python3.6/site-packages/airflow/contrib/operators/kubernetes_pod_operator.py",
line 194, in execute
config_file=self.config_file)
File
"/usr/local/lib/python3.6/site-packages/airflow/contrib/kubernetes/kube_client.py",
line 103, in get_kube_client
client_conf = _get_kube_config(in_cluster, cluster_context, config_file)
File
"/usr/local/lib/python3.6/site-packages/airflow/contrib/kubernetes/kube_client.py",
line 41, in _get_kube_config
config.load_incluster_config()
File
"/usr/local/lib/python3.6/site-packages/kubernetes/config/incluster_config.py",
line 96, in load_incluster_config
cert_filename=SERVICE_CERT_FILENAME).load_and_set()
File
"/usr/local/lib/python3.6/site-packages/kubernetes/config/incluster_config.py",
line 47, in load_and_set
self._load_config()
File
"/usr/local/lib/python3.6/site-packages/kubernetes/config/incluster_config.py",
line 53, in _load_config
raise ConfigException("Service host/port is not set.")
was:
Hi
In 10.6, the podoperator works with the code below. Airflow is running in a
docker container, with a valid kubernetes config file. In 1.10.7 the code fails
with various
------------------------------------------------------------------------------
from airflow import DAG
from datetime import datetime, timedelta
from airflow.contrib.operators.kubernetes_pod_operator import
KubernetesPodOperator
from airflow.operators.dummy_operator import DummyOperator
default_args = {
'owner': 'mikep',
'depends_on_past': False,
'start_date': datetime(2020, 1, 22),
'retries': 0,
'retry_delay': timedelta(minutes=1)
}
dag = DAG('podtest1',
default_args=default_args,
catchup=False,
schedule_interval=None)
doPod = KubernetesPodOperator(
namespace="mynamespace",
image="mypythonimage",
is_delete_operator_pod=True,
cmds=["python","-c"],
arguments=["print('hello world')"],
name="mike-pod-operator",
task_id="pod-operator",
config_file="/app/config",
get_logs=True,
dag=dag
---------------------------------------------------------------
[2020-01-23 07:26:23,245] \{logging_mixin.py:112} INFO - Running %s on host %s
<TaskInstance: podtest1.pod-operator 2020-01-23T13:26:16.911219+00:00
[running]> mypc
[2020-01-23 07:26:23,266] \{taskinstance.py:1088} ERROR - Service host/port is
not set.
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/airflow/models/taskinstance.py",
line 955, in _run_raw_task
result = task_copy.execute(context=context)
File
"/usr/local/lib/python3.6/site-packages/airflow/contrib/operators/kubernetes_pod_operator.py",
line 194, in execute
config_file=self.config_file)
File
"/usr/local/lib/python3.6/site-packages/airflow/contrib/kubernetes/kube_client.py",
line 103, in get_kube_client
client_conf = _get_kube_config(in_cluster, cluster_context, config_file)
File
"/usr/local/lib/python3.6/site-packages/airflow/contrib/kubernetes/kube_client.py",
line 41, in _get_kube_config
config.load_incluster_config()
File
"/usr/local/lib/python3.6/site-packages/kubernetes/config/incluster_config.py",
line 96, in load_incluster_config
cert_filename=SERVICE_CERT_FILENAME).load_and_set()
File
"/usr/local/lib/python3.6/site-packages/kubernetes/config/incluster_config.py",
line 47, in load_and_set
self._load_config()
File
"/usr/local/lib/python3.6/site-packages/kubernetes/config/incluster_config.py",
line 53, in _load_config
raise ConfigException("Service host/port is not set.")
> KubernetesPodOperator not working in 10.7
> -----------------------------------------
>
> Key: AIRFLOW-6621
> URL: https://issues.apache.org/jira/browse/AIRFLOW-6621
> Project: Apache Airflow
> Issue Type: Bug
> Components: operators
> Affects Versions: 1.10.7
> Environment: kubernetes cluster. Airflow running outside cluster in a
> docker container on my pc.
> Reporter: Mike Prior
> Priority: Major
>
> Hi
> In 10.6, the podoperator works with the code below. Airflow is running in a
> docker container, with a valid kubernetes config file. In 1.10.7 the code
> fails with various exceptions relating to reading the config.
> ------------------------------------------------------------------------------
>
> from airflow import DAG
> from datetime import datetime, timedelta
> from airflow.contrib.operators.kubernetes_pod_operator import
> KubernetesPodOperator
> from airflow.operators.dummy_operator import DummyOperator
> default_args =
> { 'owner': 'mikep', 'depends_on_past': False, 'start_date': datetime(2020, 1,
> 22), 'retries': 0, 'retry_delay': timedelta(minutes=1) }
>
> dag = DAG('podtest1',
> default_args=default_args,
> catchup=False,
> schedule_interval=None)
> doPod = KubernetesPodOperator(
> namespace="mynamespace",
> image="mypythonimage",
> is_delete_operator_pod=True,
> cmds=["python","-c"],
> arguments=["print('hello world')"],
> name="mike-pod-operator",
> task_id="pod-operator",
> config_file="/app/config",
> get_logs=True,
> dag=dag
>
> ---------------------------------------------------------------
>
> [2020-01-23 07:26:23,245] \{logging_mixin.py:112} INFO - Running %s on host
> %s <TaskInstance: podtest1.pod-operator 2020-01-23T13:26:16.911219+00:00
> [running]> mypc
> [2020-01-23 07:26:23,266] \{taskinstance.py:1088} ERROR - Service host/port
> is not set.
> Traceback (most recent call last):
> File
> "/usr/local/lib/python3.6/site-packages/airflow/models/taskinstance.py", line
> 955, in _run_raw_task
> result = task_copy.execute(context=context)
> File
> "/usr/local/lib/python3.6/site-packages/airflow/contrib/operators/kubernetes_pod_operator.py",
> line 194, in execute
> config_file=self.config_file)
> File
> "/usr/local/lib/python3.6/site-packages/airflow/contrib/kubernetes/kube_client.py",
> line 103, in get_kube_client
> client_conf = _get_kube_config(in_cluster, cluster_context, config_file)
> File
> "/usr/local/lib/python3.6/site-packages/airflow/contrib/kubernetes/kube_client.py",
> line 41, in _get_kube_config
> config.load_incluster_config()
> File
> "/usr/local/lib/python3.6/site-packages/kubernetes/config/incluster_config.py",
> line 96, in load_incluster_config
> cert_filename=SERVICE_CERT_FILENAME).load_and_set()
> File
> "/usr/local/lib/python3.6/site-packages/kubernetes/config/incluster_config.py",
> line 47, in load_and_set
> self._load_config()
> File
> "/usr/local/lib/python3.6/site-packages/kubernetes/config/incluster_config.py",
> line 53, in _load_config
> raise ConfigException("Service host/port is not set.")
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
