akshaychitneni commented on code in PR #2244: URL: https://github.com/apache/datafusion-ballista/pull/2244#discussion_r3760102103
########## chaos-testing/src/k8s.rs: ########## @@ -0,0 +1,526 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A Kubernetes (kind) backend for the chaos harness. +//! +//! Where [`crate::cluster::TestCluster`] spawns the scheduler and executors as +//! local OS processes, [`K8sCluster`] runs them as pods in a `kind` cluster: the +//! scheduler as a Deployment behind a `ClusterIP` Service, and the executors as +//! a labelled Deployment (so a later scenario can `scale`/`delete pod` them). +//! +//! The fixture is shared through a `hostPath` volume mounted into both pods. +//! The harness writes the parquet under [`fixture_dir`] on the host; kind's +//! `extraMounts` bind that path into the node, and each pod `hostPath`-mounts +//! it, so the path string is identical on host, node, and pod and the +//! schema-inferring `CREATE EXTERNAL TABLE ... LOCATION` that +//! `Fixture::register_sql` emits resolves the same everywhere — no object store. +//! The directory lives under `$HOME` rather than `/tmp`: Docker Desktop reliably +//! shares the home directory into its VM (and thus the kind node), whereas the +//! VM's `/tmp` is not the host's. Because kind is single-node, a rescheduled pod +//! re-mounts the same directory, so the fixture survives executor kills. +//! +//! The harness process runs outside the cluster, so it reaches the scheduler's +//! gRPC + REST (both on one port) through a `kubectl port-forward`. Results are +//! fetched through the scheduler's embedded flight proxy +//! (`advertise_flight_sql_endpoint`), so the client never contacts executor pod +//! IPs directly. +//! +//! This backend shells out to `kubectl`; it assumes a `kind` cluster already +//! exists, `kubectl` is on `PATH` pointed at it, and the chaos image has been +//! `kind load`ed. See `chaos-testing/k8s/` and the crate README for the runbook. + +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +/// Directory shared between the harness and the pods, holding the fixture. +/// +/// Defaults to `$HOME/.ballista-chaos-fixtures` and can be overridden with +/// `CHAOS_FIXTURE_DIR` (the run script sets it so the kind `extraMounts` and +/// this backend agree). A path under `$HOME` is used rather than `/tmp` because +/// Docker Desktop reliably shares the home directory into its VM (and thus into +/// the kind node), whereas the VM has its own `/tmp` that is not the host's. +/// The path is identical on host, node, and pod, so the schema-inferring +/// `CREATE EXTERNAL TABLE ... LOCATION` that `Fixture::register_sql` emits +/// resolves the same everywhere. +fn fixture_dir() -> String { + std::env::var("CHAOS_FIXTURE_DIR").unwrap_or_else(|_| { + let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()); + format!("{home}/.ballista-chaos-fixtures") + }) +} + +const CHAOS_IMAGE: &str = "ballista-chaos:test"; +const SCHEDULER_PORT: u16 = 50050; +const EXECUTOR_DEPLOYMENT: &str = "ballista-executor"; + +/// How an executor pod is removed. +#[derive(Clone, Copy, Debug)] +pub enum KillMode { + /// `kubectl delete pod` — SIGTERM plus the termination grace period, so the + /// executor's graceful-shutdown path runs (the path a raw process `SIGKILL` + /// can never reach). + Graceful, + /// `kubectl delete pod --grace-period=0 --force` — an abrupt loss, the + /// closest k8s analogue of the process harness's `SIGKILL`. + Forced, +} + +/// A Ballista cluster running as pods in a kind cluster. +pub struct K8sCluster { + namespace: String, + scheduler_local_port: u16, + port_forward: Child, + shared_dir: PathBuf, +} + +impl K8sCluster { + /// Deploy a scheduler + `executors` executor pods, wait until all executors + /// have registered, and open a port-forward to the scheduler. + pub async fn start(executors: usize) -> Result<Self, String> { + require_kubectl()?; + + // One cluster per process; --test-threads=1 keeps it to one at a time. + let namespace = format!("chaos-{}", std::process::id()); Review Comment: Fixed. The name is now chaos-{pid}-{counter} via an atomic NS_SEQ, so each K8sCluster::start gets a distinct namespace -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
