amoghrajesh commented on code in PR #61394:
URL: https://github.com/apache/airflow/pull/61394#discussion_r2780964879
##########
task-sdk/src/airflow/sdk/execution_time/task_runner.py:
##########
@@ -957,13 +957,35 @@ def sort_dict_recursively(obj: Any) -> Any:
return tuple(sort_dict_recursively(item) for item in obj)
return obj
+ def to_serializable(obj: Any) -> Any:
+ """Convert objects to JSON serializable types."""
+ if obj is None or isinstance(obj, (str, int, float, bool)):
+ return obj
+
+ if isinstance(obj, dict):
+ return {k: to_serializable(v) for k, v in obj.items()}
+
+ if isinstance(obj, (list, tuple)):
+ return [to_serializable(item) for item in obj]
+
+ # Some objects like Kubernetes ones or boto3 ones have to_dict
defined, try using them if they exist
+ if hasattr(obj, "to_dict"):
+ return obj.to_dict()
+
+ return obj
Review Comment:
Good idea. I thought about it but didn't think `default` would handle the
nested objects well. Performed a small test and I like that it does it:
```python
def convert(obj):
return json.dumps(to_serializable(obj))
def fallack(obj):
return json.dumps(obj, default=_fallback_serialization)
fallack(env_vars)
Out[15]: '[{"name": "var1", "value": "This is a test phrase.", "value_from":
null}, {"name": "var2", "value": "secret1", "value_from": null}, {"name":
"var3", "value": "secret2", "value_from": null}]'
convert(env_vars)
Out[16]: '[{"name": "var1", "value": "This is a test phrase.", "value_from":
null}, {"name": "var2", "value": "secret1", "value_from": null}, {"name":
"var3", "value": "secret2", "value_from": null}]'
fallack(env_vars) == convert(env_vars)
Out[17]: True
pod_spec = k8s.V1PodSpec(
containers=[
k8s.V1Container(
name="my-container",
image="nginx",
env=[
k8s.V1EnvVar(name="SECRET1", value="my secret value"),
k8s.V1EnvVar(name="SECRET2", value="another secret"),
],
volume_mounts=[
k8s.V1VolumeMount(name="data", mount_path="/data")
]
)
],
volumes=[
k8s.V1Volume(
name="data",
persistent_volume_claim=k8s.V1PersistentVolumeClaimVolumeSource(
claim_name="my-pvc"
)
)
]
)
fallack(pod_spec)
Out[19]: '{"active_deadline_seconds": null, "affinity": null,
"automount_service_account_token": null, "containers": [{"args": null,
"command": null, "env": [{"name": "SECRET1", "value": "my secret value",
"value_from": null}, {"name": "SECRET2", "value": "another secret",
"value_from": null}], "env_from": null, "image": "nginx", "image_pull_policy":
null, "lifecycle": null, "liveness_probe": null, "name": "my-container",
"ports": null, "readiness_probe": null, "resize_policy": null, "resources":
null, "restart_policy": null, "restart_policy_rules": null, "security_context":
null, "startup_probe": null, "stdin": null, "stdin_once": null,
"termination_message_path": null, "termination_message_policy": null, "tty":
null, "volume_devices": null, "volume_mounts": [{"mount_path": "/data",
"mount_propagation": null, "name": "data", "read_only": null,
"recursive_read_only": null, "sub_path": null, "sub_path_expr": null}],
"working_dir": null}], "dns_config": null, "dns_policy": null, "en
able_service_links": null, "ephemeral_containers": null, "host_aliases": null,
"host_ipc": null, "host_network": null, "host_pid": null, "host_users": null,
"hostname": null, "hostname_override": null, "image_pull_secrets": null,
"init_containers": null, "node_name": null, "node_selector": null, "os": null,
"overhead": null, "preemption_policy": null, "priority": null,
"priority_class_name": null, "readiness_gates": null, "resource_claims": null,
"resources": null, "restart_policy": null, "runtime_class_name": null,
"scheduler_name": null, "scheduling_gates": null, "security_context": null,
"service_account": null, "service_account_name": null, "set_hostname_as_fqdn":
null, "share_process_namespace": null, "subdomain": null,
"termination_grace_period_seconds": null, "tolerations": null,
"topology_spread_constraints": null, "volumes": [{"aws_elastic_block_store":
null, "azure_disk": null, "azure_file": null, "cephfs": null, "cinder": null,
"config_map": null, "csi": null, "downward_a
pi": null, "empty_dir": null, "ephemeral": null, "fc": null, "flex_volume":
null, "flocker": null, "gce_persistent_disk": null, "git_repo": null,
"glusterfs": null, "host_path": null, "image": null, "iscsi": null, "name":
"data", "nfs": null, "persistent_volume_claim": {"claim_name": "my-pvc",
"read_only": null}, "photon_persistent_disk": null, "portworx_volume": null,
"projected": null, "quobyte": null, "rbd": null, "scale_io": null, "secret":
null, "storageos": null, "vsphere_volume": null}], "workload_ref": null}'
convert(pod_spec)
Out[20]: '{"active_deadline_seconds": null, "affinity": null,
"automount_service_account_token": null, "containers": [{"args": null,
"command": null, "env": [{"name": "SECRET1", "value": "my secret value",
"value_from": null}, {"name": "SECRET2", "value": "another secret",
"value_from": null}], "env_from": null, "image": "nginx", "image_pull_policy":
null, "lifecycle": null, "liveness_probe": null, "name": "my-container",
"ports": null, "readiness_probe": null, "resize_policy": null, "resources":
null, "restart_policy": null, "restart_policy_rules": null, "security_context":
null, "startup_probe": null, "stdin": null, "stdin_once": null,
"termination_message_path": null, "termination_message_policy": null, "tty":
null, "volume_devices": null, "volume_mounts": [{"mount_path": "/data",
"mount_propagation": null, "name": "data", "read_only": null,
"recursive_read_only": null, "sub_path": null, "sub_path_expr": null}],
"working_dir": null}], "dns_config": null, "dns_policy": null, "en
able_service_links": null, "ephemeral_containers": null, "host_aliases": null,
"host_ipc": null, "host_network": null, "host_pid": null, "host_users": null,
"hostname": null, "hostname_override": null, "image_pull_secrets": null,
"init_containers": null, "node_name": null, "node_selector": null, "os": null,
"overhead": null, "preemption_policy": null, "priority": null,
"priority_class_name": null, "readiness_gates": null, "resource_claims": null,
"resources": null, "restart_policy": null, "runtime_class_name": null,
"scheduler_name": null, "scheduling_gates": null, "security_context": null,
"service_account": null, "service_account_name": null, "set_hostname_as_fqdn":
null, "share_process_namespace": null, "subdomain": null,
"termination_grace_period_seconds": null, "tolerations": null,
"topology_spread_constraints": null, "volumes": [{"aws_elastic_block_store":
null, "azure_disk": null, "azure_file": null, "cephfs": null, "cinder": null,
"config_map": null, "csi": null, "downward_a
pi": null, "empty_dir": null, "ephemeral": null, "fc": null, "flex_volume":
null, "flocker": null, "gce_persistent_disk": null, "git_repo": null,
"glusterfs": null, "host_path": null, "image": null, "iscsi": null, "name":
"data", "nfs": null, "persistent_volume_claim": {"claim_name": "my-pvc",
"read_only": null}, "photon_persistent_disk": null, "portworx_volume": null,
"projected": null, "quobyte": null, "rbd": null, "scale_io": null, "secret":
null, "storageos": null, "vsphere_volume": null}], "workload_ref": null}'
fallack(pod_spec) == convert(pod_spec)
Out[21]: True
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
