repl-mike-roest opened a new issue, #21943:
URL: https://github.com/apache/airflow/issues/21943
### Official Helm Chart version
1.4.0 (latest released)
### Apache Airflow version
2.2.4 (latest released)
### Kubernetes Version
1.21.5 (EKS)
### Helm Chart configuration
We are using a external RDS DB server configured via secrets.
Also have specified
```
airflowVersion: 2.2.3
defaultAirflowTag: 2.2.3
```
Along with these flags As without them while deploying via codebuild the
chart was never progressing as the create-user/run-db-migrations jobs were not
running
```
createUserJob:
useHelmHooks: false
migrateDatabaseJob:
useHelmHooks: false
```
### Docker Image customisations
Happens both with a transition from airflow default image 2.2.3 -> 2.2.4 and
with changing our custom image between versions or from a default airflow image
to our custom image
### What happened
The following error was returned from the helm upgrade command
```
Error: UPGRADE FAILED: cannot patch "pre-production-create-user" with kind
Job: Job.batch "pre-production-create-user" is invalid: spec.template: Invalid
value: core.PodTemplateSpec{ObjectMeta:v1.ObjectMeta{Name:"", GenerateName:"",
Namespace:"", SelfLink:"", UID:"", ResourceVersion:"", Generation:0,
CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0,
loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil),
DeletionGracePeriodSeconds:(*int64)(nil),
Labels:map[string]string{"component":"create-user-job",
"controller-uid":"52e67857-b3f0-414c-b176-3027c93e4a05",
"job-name":"pre-production-create-user", "release":"pre-production",
"tier":"airflow"}, Annotations:map[string]string(nil),
OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil),
ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)},
Spec:core.PodSpec{Volumes:[]core.Volume{core.Volume{Name:"config",
VolumeSource:core.VolumeSource{HostPath:(*core.HostPathVolumeSource)(nil),
EmptyDir:(*core.Empty
DirVolumeSource)(nil),
GCEPersistentDisk:(*core.GCEPersistentDiskVolumeSource)(nil),
AWSElasticBlockStore:(*core.AWSElasticBlockStoreVolumeSource)(nil),
GitRepo:(*core.GitRepoVolumeSource)(nil),
Secret:(*core.SecretVolumeSource)(nil), NFS:(*core.NFSVolumeSource)(nil),
ISCSI:(*core.ISCSIVolumeSource)(nil),
Glusterfs:(*core.GlusterfsVolumeSource)(nil),
PersistentVolumeClaim:(*core.PersistentVolumeClaimVolumeSource)(nil),
RBD:(*core.RBDVolumeSource)(nil), Quobyte:(*core.QuobyteVolumeSource)(nil),
FlexVolume:(*core.FlexVolumeSource)(nil),
Cinder:(*core.CinderVolumeSource)(nil), CephFS:(*core.CephFSVolumeSource)(nil),
Flocker:(*core.FlockerVolumeSource)(nil),
DownwardAPI:(*core.DownwardAPIVolumeSource)(nil),
FC:(*core.FCVolumeSource)(nil), AzureFile:(*core.AzureFileVolumeSource)(nil),
ConfigMap:(*core.ConfigMapVolumeSource)(0xc0107dfdc0),
VsphereVolume:(*core.VsphereVirtualDiskVolumeSource)(nil),
AzureDisk:(*core.AzureDiskVolumeSource)(nil),
PhotonPersistentDisk:(*core.PhotonPersistentDi
skVolumeSource)(nil), Projected:(*core.ProjectedVolumeSource)(nil),
PortworxVolume:(*core.PortworxVolumeSource)(nil),
ScaleIO:(*core.ScaleIOVolumeSource)(nil),
StorageOS:(*core.StorageOSVolumeSource)(nil), CSI:(*core.CSIVolumeSource)(nil),
Ephemeral:(*core.EphemeralVolumeSource)(nil)}}},
InitContainers:[]core.Container(nil),
Containers:[]core.Container{core.Container{Name:"create-user",
Image:"434423891815.dkr.ecr.us-west-2.amazonaws.com/airflow-playground/airflow:b-23-IP2-51",
Command:[]string(nil), Args:[]string{"bash", "-c", "airflow users create
\"$@\"", "--", "-r", "Admin", "-u", "admin", "-e", "[email protected]", "-f",
"admin", "-l", "user", "-p", "DFPGxku#V#&h{C:)qiOmta3s"}, WorkingDir:"",
Ports:[]core.ContainerPort(nil), EnvFrom:[]core.EnvFromSource{},
Env:[]core.EnvVar{core.EnvVar{Name:"AIRFLOW__CORE__FERNET_KEY", Value:"",
ValueFrom:(*core.EnvVarSource)(0xc016c97560)},
core.EnvVar{Name:"AIRFLOW__CORE__SQL_ALCHEMY_CONN", Value:"",
ValueFrom:(*core.EnvVarSource)(0xc016c9758
0)}, core.EnvVar{Name:"AIRFLOW_CONN_AIRFLOW_DB", Value:"",
ValueFrom:(*core.EnvVarSource)(0xc016c975a0)},
core.EnvVar{Name:"AIRFLOW__WEBSERVER__SECRET_KEY", Value:"",
ValueFrom:(*core.EnvVarSource)(0xc016c975c0)},
core.EnvVar{Name:"AIRFLOW__CELERY__CELERY_RESULT_BACKEND", Value:"",
ValueFrom:(*core.EnvVarSource)(0xc016c97600)},
core.EnvVar{Name:"AIRFLOW__CELERY__RESULT_BACKEND", Value:"",
ValueFrom:(*core.EnvVarSource)(0xc016c97620)},
core.EnvVar{Name:"AIRFLOW__CELERY__BROKER_URL", Value:"",
ValueFrom:(*core.EnvVarSource)(0xc016c97640)}},
Resources:core.ResourceRequirements{Limits:core.ResourceList(nil),
Requests:core.ResourceList(nil)},
VolumeMounts:[]core.VolumeMount{core.VolumeMount{Name:"config", ReadOnly:true,
MountPath:"/opt/airflow/airflow.cfg", SubPath:"airflow.cfg",
MountPropagation:(*core.MountPropagationMode)(nil), SubPathExpr:""}},
VolumeDevices:[]core.VolumeDevice(nil), LivenessProbe:(*core.Probe)(nil),
ReadinessProbe:(*core.Probe)(nil), StartupProbe:(*core.Probe)(nil),
Lifecycle:(*core.Lifecycle)(nil),
TerminationMessagePath:"/dev/termination-log", TerminationMessagePolicy:"File",
ImagePullPolicy:"IfNotPresent", SecurityContext:(*core.SecurityContext)(nil),
Stdin:false, StdinOnce:false, TTY:false}},
EphemeralContainers:[]core.EphemeralContainer(nil), RestartPolicy:"OnFailure",
TerminationGracePeriodSeconds:(*int64)(0xc01a548358),
ActiveDeadlineSeconds:(*int64)(nil), DNSPolicy:"ClusterFirst",
NodeSelector:map[string]string{},
ServiceAccountName:"pre-production-airflow-create-user-job",
AutomountServiceAccountToken:(*bool)(nil), NodeName:"",
SecurityContext:(*core.PodSecurityContext)(0xc01771ca80),
ImagePullSecrets:[]core.LocalObjectReference(nil), Hostname:"", Subdomain:"",
SetHostnameAsFQDN:(*bool)(nil), Affinity:(*core.Affinity)(0xc01b38ff20),
SchedulerName:"default-scheduler", Tolerations:[]core.Toleration{},
HostAliases:[]core.HostAlias(nil), PriorityClassName:"",
Priority:(*int32)(nil), PreemptionPolicy:(*core.PreemptionPolicy)(nil), DNSConfi
g:(*core.PodDNSConfig)(nil), ReadinessGates:[]core.PodReadinessGate(nil),
RuntimeClassName:(*string)(nil), Overhead:core.ResourceList(nil),
EnableServiceLinks:(*bool)(nil),
TopologySpreadConstraints:[]core.TopologySpreadConstraint(nil)}}: field is
immutable && cannot patch "pre-production-run-airflow-migrations" with kind
Job: Job.batch "pre-production-run-airflow-migrations" is invalid:
spec.template: Invalid value:
core.PodTemplateSpec{ObjectMeta:v1.ObjectMeta{Name:"", GenerateName:"",
Namespace:"", SelfLink:"", UID:"", ResourceVersion:"", Generation:0,
CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0,
loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil),
DeletionGracePeriodSeconds:(*int64)(nil),
Labels:map[string]string{"component":"run-airflow-migrations",
"controller-uid":"19d78edd-2df2-4f61-ba6b-01592d103327",
"job-name":"pre-production-run-airflow-migrations", "release":"pre-production",
"tier":"airflow"}, Annotations:map[string]string(nil), OwnerReferences:
[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"",
ManagedFields:[]v1.ManagedFieldsEntry(nil)},
Spec:core.PodSpec{Volumes:[]core.Volume{core.Volume{Name:"config",
VolumeSource:core.VolumeSource{HostPath:(*core.HostPathVolumeSource)(nil),
EmptyDir:(*core.EmptyDirVolumeSource)(nil),
GCEPersistentDisk:(*core.GCEPersistentDiskVolumeSource)(nil),
AWSElasticBlockStore:(*core.AWSElasticBlockStoreVolumeSource)(nil),
GitRepo:(*core.GitRepoVolumeSource)(nil),
Secret:(*core.SecretVolumeSource)(nil), NFS:(*core.NFSVolumeSource)(nil),
ISCSI:(*core.ISCSIVolumeSource)(nil),
Glusterfs:(*core.GlusterfsVolumeSource)(nil),
PersistentVolumeClaim:(*core.PersistentVolumeClaimVolumeSource)(nil),
RBD:(*core.RBDVolumeSource)(nil), Quobyte:(*core.QuobyteVolumeSource)(nil),
FlexVolume:(*core.FlexVolumeSource)(nil),
Cinder:(*core.CinderVolumeSource)(nil), CephFS:(*core.CephFSVolumeSource)(nil),
Flocker:(*core.FlockerVolumeSource)(nil),
DownwardAPI:(*core.DownwardAPIVolumeSource)(nil), FC:(*cor
e.FCVolumeSource)(nil), AzureFile:(*core.AzureFileVolumeSource)(nil),
ConfigMap:(*core.ConfigMapVolumeSource)(0xc01a11e200),
VsphereVolume:(*core.VsphereVirtualDiskVolumeSource)(nil),
AzureDisk:(*core.AzureDiskVolumeSource)(nil),
PhotonPersistentDisk:(*core.PhotonPersistentDiskVolumeSource)(nil),
Projected:(*core.ProjectedVolumeSource)(nil),
PortworxVolume:(*core.PortworxVolumeSource)(nil),
ScaleIO:(*core.ScaleIOVolumeSource)(nil),
StorageOS:(*core.StorageOSVolumeSource)(nil), CSI:(*core.CSIVolumeSource)(nil),
Ephemeral:(*core.EphemeralVolumeSource)(nil)}}},
InitContainers:[]core.Container(nil),
Containers:[]core.Container{core.Container{Name:"run-airflow-migrations",
Image:"434423891815.dkr.ecr.us-west-2.amazonaws.com/airflow-playground/airflow:b-23-IP2-51",
Command:[]string(nil), Args:[]string{"bash", "-c", "airflow db upgrade"},
WorkingDir:"", Ports:[]core.ContainerPort(nil), EnvFrom:[]core.EnvFromSource{},
Env:[]core.EnvVar{core.EnvVar{Name:"AIRFLOW__CORE__FERNET_KEY", Value:"",
ValueFrom:(*core.EnvVarSource)(0xc00cd021c0)},
core.EnvVar{Name:"AIRFLOW__CORE__SQL_ALCHEMY_CONN", Value:"",
ValueFrom:(*core.EnvVarSource)(0xc00cd02200)},
core.EnvVar{Name:"AIRFLOW_CONN_AIRFLOW_DB", Value:"",
ValueFrom:(*core.EnvVarSource)(0xc00cd02220)},
core.EnvVar{Name:"AIRFLOW__WEBSERVER__SECRET_KEY", Value:"",
ValueFrom:(*core.EnvVarSource)(0xc00cd02260)},
core.EnvVar{Name:"AIRFLOW__CELERY__CELERY_RESULT_BACKEND", Value:"",
ValueFrom:(*core.EnvVarSource)(0xc00cd02280)},
core.EnvVar{Name:"AIRFLOW__CELERY__RESULT_BACKEND", Value:"",
ValueFrom:(*core.EnvVarSource)(0xc00cd022c0)},
core.EnvVar{Name:"AIRFLOW__CELERY__BROKER_URL", Value:"",
ValueFrom:(*core.EnvVarSource)(0xc00cd022e0)}},
Resources:core.ResourceRequirements{Limits:core.ResourceList(nil),
Requests:core.ResourceList(nil)},
VolumeMounts:[]core.VolumeMount{core.VolumeMount{Name:"config", ReadOnly:true,
MountPath:"/opt/airflow/airflow.cfg", SubPath:"airflow.cfg",
MountPropagation:(*core.MountPropagationMode)(nil), SubPath
Expr:""}}, VolumeDevices:[]core.VolumeDevice(nil),
LivenessProbe:(*core.Probe)(nil), ReadinessProbe:(*core.Probe)(nil),
StartupProbe:(*core.Probe)(nil), Lifecycle:(*core.Lifecycle)(nil),
TerminationMessagePath:"/dev/termination-log", TerminationMessagePolicy:"File",
ImagePullPolicy:"IfNotPresent", SecurityContext:(*core.SecurityContext)(nil),
Stdin:false, StdinOnce:false, TTY:false}},
EphemeralContainers:[]core.EphemeralContainer(nil), RestartPolicy:"OnFailure",
TerminationGracePeriodSeconds:(*int64)(0xc01bc01a08),
ActiveDeadlineSeconds:(*int64)(nil), DNSPolicy:"ClusterFirst",
NodeSelector:map[string]string{},
ServiceAccountName:"pre-production-airflow-migrate-database-job",
AutomountServiceAccountToken:(*bool)(nil), NodeName:"",
SecurityContext:(*core.PodSecurityContext)(0xc008e4f300),
ImagePullSecrets:[]core.LocalObjectReference(nil), Hostname:"", Subdomain:"",
SetHostnameAsFQDN:(*bool)(nil), Affinity:(*core.Affinity)(0xc0127b9ad0),
SchedulerName:"default-scheduler", Tolerations:[
]core.Toleration{}, HostAliases:[]core.HostAlias(nil), PriorityClassName:"",
Priority:(*int32)(nil), PreemptionPolicy:(*core.PreemptionPolicy)(nil),
DNSConfig:(*core.PodDNSConfig)(nil),
ReadinessGates:[]core.PodReadinessGate(nil), RuntimeClassName:(*string)(nil),
Overhead:core.ResourceList(nil), EnableServiceLinks:(*bool)(nil),
TopologySpreadConstraints:[]core.TopologySpreadConstraint(nil)}}: field is
immutable
```
### What you expected to happen
The helm chart to successfully upgrade and change my running images to a new
version
### How to reproduce
deploy helm chart with
```
airflowVersion: 2.2.3
defaultAirflowTag: 2.2.3
```
in your values.yaml
and the following command
helm upgrade --install --wait --timeout 900s pre-production
apache-airflow/airflow --namespace airflow --version 1.4.0 \
-f values.yaml
Then run the same command after changing the image tags to 2.2.4
### Anything else
Seems to happen whenever we change the image tag (even within the same
release) if we're using a custom image that contains our dags changing from one
tag to the other gets the same error.
### Are you willing to submit PR?
- [X] Yes I am willing to submit a PR!
### Code of Conduct
- [X] I agree to follow this project's [Code of
Conduct](https://github.com/apache/airflow/blob/main/CODE_OF_CONDUCT.md)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
