imbajin commented on code in PR #746: URL: https://github.com/apache/hugegraph-toolchain/pull/746#discussion_r3627271292
########## .codex/scripts/hubble.sh: ########## @@ -0,0 +1,473 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail +unset CDPATH + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(cd "${SCRIPT_DIR}/../.." && pwd) +WORKTREE=${CODEX_WORKTREE_PATH:-${REPO_ROOT}} +SOURCE_TREE=${CODEX_SOURCE_TREE_PATH:-${REPO_ROOT}} +NODE_HOME=${HUBBLE_NODE_HOME:-"${HOME}/.nvm/versions/node/v18.20.8"} +CODEX_HOME=${CODEX_HOME:-"${HOME}/.codex"} + +hash_text() { + printf '%s' "$1" | LC_ALL=C shasum -a 256 | awk '{print substr($1, 1, 12)}' +} + +STATE_ROOT="${CODEX_HOME}/run/toolchain-$(hash_text "${SOURCE_TREE}")" +WORKTREE_ID=$(hash_text "${WORKTREE}") +RUNTIME_DIR="${STATE_ROOT}/worktrees/${WORKTREE_ID}" +BE_RUNTIME_DIR="${STATE_ROOT}/hubble-be" + +fe_port() { + local path=${1:-${WORKTREE}} + local checksum + checksum=$(printf '%s' "${path}" | cksum | awk '{print $1}') + echo $((3001 + checksum % 99)) +} + +read_field() { + local file=$1 key=$2 + awk -F= -v key="${key}" '$1 == key {sub(/^[^=]*=/, ""); print; exit}' "${file}" +} + +process_alive() { + local pid=$1 + [[ "${pid}" =~ ^[0-9]+$ ]] && kill -0 "${pid}" 2>/dev/null +} + +process_start_time() { + ps -o lstart= -p "$1" 2>/dev/null | sed -E 's/^[[:space:]]+//;s/[[:space:]]+$//' +} + +process_is_descendant() { + local child=$1 ancestor=$2 parent + while [[ "${child}" =~ ^[0-9]+$ ]] && ((child > 1)); do + [[ "${child}" == "${ancestor}" ]] && return 0 + parent=$(ps -o ppid= -p "${child}" 2>/dev/null | tr -d ' ') + [[ -n "${parent}" && "${parent}" != "${child}" ]] || return 1 + child=${parent} + done + return 1 +} + +owner_identity_matches() { + local file=$1 pid expected_start command + pid=$(read_field "${file}" pid) + expected_start=$(read_field "${file}" start_time) + process_alive "${pid}" || return 1 + [[ -n "${expected_start}" ]] || return 1 + [[ "$(process_start_time "${pid}")" == "${expected_start}" ]] || return 1 + command=$(ps -o command= -p "${pid}" 2>/dev/null || true) + [[ "${command}" == *".codex/scripts/hubble.sh"* ]] +} + +owner_service_matches() { + local file=$1 pid port listener + owner_identity_matches "${file}" || return 1 + pid=$(read_field "${file}" pid) + port=$(read_field "${file}" port) + listener=$(lsof -tiTCP:"${port}" -sTCP:LISTEN 2>/dev/null | head -1 || true) + [[ -z "${listener}" ]] || process_is_descendant "${listener}" "${pid}" +} + +acquire_state_lock() { + local attempt probe + mkdir -p "${STATE_ROOT}" + sleep 5 & + probe=$! + STATE_LOCK_PID=$(ps -o ppid= -p "${probe}" | tr -d ' ') + kill "${probe}" 2>/dev/null || true + wait "${probe}" 2>/dev/null || true + [[ "${STATE_LOCK_PID}" =~ ^[0-9]+$ ]] || { + echo "ERROR: unable to identify owner-lock process" >&2 + return 1 + } + if command -v flock >/dev/null 2>&1; then + STATE_LOCK_KIND=flock + STATE_LOCK_PATH="${STATE_ROOT}/.owner.lock" + exec 9>"${STATE_LOCK_PATH}" + flock -w 12 9 || { + echo "ERROR: timed out waiting for Hubble owner lock" >&2 + return 1 + } + return + fi + if command -v shlock >/dev/null 2>&1; then + STATE_LOCK_KIND=shlock + STATE_LOCK_PATH="${STATE_ROOT}/.owner.lock" + for ((attempt = 0; attempt < 240; attempt++)); do + shlock -f "${STATE_LOCK_PATH}" -p "${STATE_LOCK_PID}" && return + sleep 0.05 + done + echo "ERROR: timed out waiting for Hubble owner lock" >&2 + return 1 + fi + + STATE_LOCK_KIND="mkdir" + STATE_LOCK_PATH="${STATE_ROOT}/.owner-lock" + STATE_LOCK_TOKEN="${STATE_LOCK_PID}-${RANDOM}" + for ((attempt = 0; attempt < 240; attempt++)); do + if mkdir "${STATE_LOCK_PATH}" 2>/dev/null; then + printf '%s\n' "${STATE_LOCK_TOKEN}" > "${STATE_LOCK_PATH}/owner" + return + fi + sleep 0.05 + done + echo "ERROR: owner lock is stuck; remove ${STATE_LOCK_PATH} if no action runs" >&2 + return 1 +} + +release_state_lock() { + local holder + case "${STATE_LOCK_KIND:-}" in + flock) + flock -u 9 2>/dev/null || true + exec 9>&- + ;; + shlock) + holder=$(cat "${STATE_LOCK_PATH}" 2>/dev/null || true) + [[ "${holder}" == "${STATE_LOCK_PID}" ]] && rm -f "${STATE_LOCK_PATH}" + ;; + mkdir) + holder=$(cat "${STATE_LOCK_PATH}/owner" 2>/dev/null || true) + [[ "${holder}" == "${STATE_LOCK_TOKEN}" ]] && rm -rf "${STATE_LOCK_PATH}" + ;; + esac +} + +with_state_lock() ( + acquire_state_lock + trap release_state_lock EXIT + "$@" +) + +new_token() { + printf '%s-%s-%s\n' "$$" "$(date +%s)" "${RANDOM}" +} + +write_owner() { + local file=$1 kind=$2 pid=$3 port=$4 token=$5 temp + mkdir -p "$(dirname "${file}")" + temp="${file}.tmp.${pid}.${RANDOM}" + { + echo "kind=${kind}" + echo "pid=${pid}" + echo "start_time=$(process_start_time "${pid}")" + echo "token=${token}" + echo "port=${port}" + echo "worktree=${WORKTREE}" + echo "branch=$(git -C "${WORKTREE}" branch --show-current 2>/dev/null || true)" + echo "head=$(git -C "${WORKTREE}" rev-parse --short HEAD 2>/dev/null || true)" + } > "${temp}" + mv "${temp}" "${file}" +} + +remove_owner_if_token() { + local file=$1 expected_token=$2 + [[ -f "${file}" ]] || return + if [[ "$(read_field "${file}" token)" == "${expected_token}" ]] && \ + [[ "$(read_field "${file}" pid)" == "$$" ]]; then + rm -f "${file}" + fi +} + +cleanup_claim() { + [[ -n "${SERVICE_OWNER:-}" && -n "${SERVICE_TOKEN:-}" ]] || return + [[ -f "${SERVICE_OWNER}" ]] || return + [[ "$(read_field "${SERVICE_OWNER}" token)" == "${SERVICE_TOKEN}" ]] || return + with_state_lock remove_owner_if_token "${SERVICE_OWNER}" "${SERVICE_TOKEN}" || true +} + +kill_tree() { + local pid=$1 child + while read -r child; do + [[ -n "${child}" ]] && kill_tree "${child}" + done < <(pgrep -P "${pid}" 2>/dev/null || true) + kill -TERM "${pid}" 2>/dev/null || true +} + +live_fe_count() { + local file pid count=0 + shopt -s nullglob + for file in "${STATE_ROOT}"/fe-*.owner; do + pid=$(read_field "${file}" pid) + if owner_service_matches "${file}"; then + count=$((count + 1)) + else + echo "WARN: removing stale FE owner for pid ${pid}" >&2 + rm -f "${file}" + fi + done + echo "${count}" +} + +claim_fe() { + local token=$1 preferred offset port owner pid owner_worktree listener + preferred=$(fe_port) + for ((offset = 0; offset < 99; offset++)); do + port=$((3001 + (preferred - 3001 + offset) % 99)) + owner="${STATE_ROOT}/fe-${port}.owner" + if [[ -f "${owner}" ]]; then + pid=$(read_field "${owner}" pid) + owner_worktree=$(read_field "${owner}" worktree) + if owner_service_matches "${owner}"; then + if [[ "${owner_worktree}" == "${WORKTREE}" ]]; then + echo "ERROR: FE already runs from this worktree on port ${port}" >&2 + return 1 + fi + continue + fi + echo "WARN: removing stale FE owner for pid ${pid}" >&2 + rm -f "${owner}" + fi + listener=$(lsof -tiTCP:"${port}" -sTCP:LISTEN 2>/dev/null | head -1 || true) + [[ -z "${listener}" ]] || continue + write_owner "${owner}" fe "$$" "${port}" "${token}" + echo "${port}" + return + done + echo "ERROR: no free Hubble FE port in 3001-3099" >&2 + return 1 +} + +start_fe() { + local port owner count token + mkdir -p "${RUNTIME_DIR}" + token=$(new_token) + port=$(with_state_lock claim_fe "${token}") + owner="${STATE_ROOT}/fe-${port}.owner" + SERVICE_OWNER=${owner} + SERVICE_TOKEN=${token} + trap cleanup_claim EXIT + + count=$(with_state_lock live_fe_count) + if ((count > 3)); then + echo "WARN: ${count} Hubble frontend servers are already running" >&2 + fi + [[ -x "${NODE_HOME}/bin/node" ]] || { + echo "ERROR: missing Node 18.20.8 at ${NODE_HOME}" >&2 + exit 1 + } + [[ -d "${WORKTREE}/hugegraph-hubble/hubble-fe/node_modules" ]] || { + echo "ERROR: frontend dependencies are missing; run the environment setup" >&2 + exit 1 + } + + export PATH="${NODE_HOME}/bin:${PATH}" + cd "${WORKTREE}/hugegraph-hubble/hubble-fe" + echo "[hubble] FE ${WORKTREE} -> http://127.0.0.1:${port}"; + PORT="${port}" BROWSER=none yarn start +} + +java11_home() { + if [[ -n "${JAVA11_HOME:-}" ]] && [[ -x "${JAVA11_HOME}/bin/java" ]]; then + echo "${JAVA11_HOME}" + elif [[ -x /usr/libexec/java_home ]]; then + /usr/libexec/java_home -v 11 2>/dev/null + elif command -v java >/dev/null 2>&1 && \ + java -version 2>&1 | head -1 | grep -Eq 'version "11\.'; then + cd "$(dirname "$(command -v java)")/.." && pwd + else + echo "ERROR: Java 11 is required; set JAVA11_HOME" >&2 + return 1 + fi +} + +maven_runner() { + local java_home=$1 candidate + for candidate in "${HUBBLE_MVND_BIN:-}" \ + /opt/homebrew/opt/mvnd@1/bin/mvnd \ + /usr/local/opt/mvnd@1/bin/mvnd \ + "$(command -v mvnd 2>/dev/null || true)"; do + if [[ -n "${candidate}" ]] && [[ -x "${candidate}" ]] && \ + JAVA_HOME="${java_home}" "${candidate}" --version >/dev/null 2>&1; then + echo "${candidate}" + return + fi + done + echo mvn +} + +loader_home() { + local candidate + if [[ -n "${HUBBLE_LOADER_HOME:-}" ]]; then + echo "${HUBBLE_LOADER_HOME}" + return + fi + candidate=$(find "${SOURCE_TREE}/hugegraph-loader" -maxdepth 1 -type d \ + -name 'apache-hugegraph-loader-*' | sort | tail -1) + if [[ -z "${candidate}" ]]; then + echo "ERROR: no packaged Loader found; set HUBBLE_LOADER_HOME" >&2 + exit 1 + fi + echo "${candidate}" +} + +claim_be() { + local switch=$1 token=$2 owner="${STATE_ROOT}/be.owner" + local pid owner_worktree listener wait_for + if [[ -f "${owner}" ]]; then + pid=$(read_field "${owner}" pid) + owner_worktree=$(read_field "${owner}" worktree) + if owner_service_matches "${owner}"; then + if [[ "${owner_worktree}" == "${WORKTREE}" ]]; then + echo "[hubble] BE already runs from this worktree (pid ${pid})" >&2 + return 10 + fi + if [[ "${switch}" != true ]]; then + echo "ERROR: BE is owned by ${owner_worktree} (pid ${pid})" >&2 + echo "Use the 'Hubble BE Switch' action to replace it." >&2 + return 1 + fi + rm -f "${owner}" + kill_tree "${pid}" + wait_for=0 + while process_alive "${pid}" && ((wait_for < 50)); do + sleep 0.1 + wait_for=$((wait_for + 1)) + done + if process_alive "${pid}"; then + echo "ERROR: previous BE pid ${pid} did not stop" >&2 + return 1 + fi + else + echo "WARN: refusing to signal stale or mismatched BE pid ${pid}" >&2 + rm -f "${owner}" + fi + fi + + listener=$(lsof -tiTCP:8088 -sTCP:LISTEN 2>/dev/null | head -1 || true) + if [[ -n "${listener}" ]]; then + echo "ERROR: unowned process ${listener} is listening on port 8088" >&2 + return 1 + fi + write_owner "${owner}" be "$$" 8088 "${token}" +} + +start_be() { + local switch=${1:-false} + local owner="${STATE_ROOT}/be.owner" token claim_status + local java_home runner classpath loader config_arg=() + mkdir -p "${RUNTIME_DIR}" + mkdir -p "${BE_RUNTIME_DIR}/logs" "${BE_RUNTIME_DIR}/upload-files" + token=$(new_token) + if with_state_lock claim_be "${switch}" "${token}"; then + : + else + claim_status=$? + [[ "${claim_status}" -eq 10 ]] && return + return "${claim_status}" + fi + SERVICE_OWNER=${owner} + SERVICE_TOKEN=${token} + trap cleanup_claim EXIT + + java_home=$(java11_home) + runner=$(maven_runner "${java_home}") + classpath="${RUNTIME_DIR}/hubble-be.classpath" + echo "[hubble] compiling BE with Java 11 + ${runner}" + ( + cd "${WORKTREE}/hugegraph-hubble" Review Comment: ‼️ A clean Codex worktree cannot start this BE action. This build only includes `hubble-be`, while that module depends on the current `hugegraph-loader`/client artifacts and `loader_home()` later requires an ignored `hugegraph-loader/apache-hugegraph-loader-*` distribution. `setup.sh` only installs frontend dependencies, so a fresh clone has neither prerequisite (the existing Hubble CI explicitly installs client+loader first). Please build/install the required reactor modules and produce/select the current Loader distribution deterministically, or provision a declared artifact, and cover `be-start` from a clean checkout. ########## .codex/tests/environment_test.sh: ########## @@ -0,0 +1,185 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail +unset CDPATH + +CODEX_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +REPO_ROOT=$(cd "${CODEX_DIR}/.." && pwd) +TEST_TMP=$(mktemp -d "${TMPDIR:-/tmp}/toolchain-env-test.XXXXXX") +TEST_TMP=$(cd "${TEST_TMP}" && pwd) +trap 'rm -rf "${TEST_TMP}"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +assert_contains() { + local actual=$1 + local expected=$2 + [[ "${actual}" == *"${expected}"* ]] || + fail "expected '${actual}' to contain '${expected}'" +} + +test_environment_toml() { + python3 - "${CODEX_DIR}/environments/environment.toml" <<'PY' +import pathlib +import sys +import tomllib + +path = pathlib.Path(sys.argv[1]) +with path.open("rb") as stream: + config = tomllib.load(stream) + +assert config["version"] == 1 +assert config["name"] == "toolchain-hubble" +assert config["setup"]["script"] +assert config["cleanup"]["script"] +names = {action["name"] for action in config["actions"]} +required = { + "Hubble FE Dev", + "Hubble BE Dev", + "Hubble Status", + "Hubble Stop", + "HStore Low Memory", + "Infrastructure Status", + "Infrastructure Stop", + "Infrastructure Reset", +} +assert required <= names, required - names +PY +} + +test_setup_dry_run() { + local output + output=$(CODEX_WORKTREE_PATH="${REPO_ROOT}" \ + "${CODEX_DIR}/scripts/setup.sh" --dry-run) + assert_contains "${output}" "Node 18.20.8" + assert_contains "${output}" "yarn install --frozen-lockfile" +} + +test_stable_frontend_ports() { + local first second + first=$("${CODEX_DIR}/scripts/hubble.sh" fe-port "/tmp/worktree-a") + second=$("${CODEX_DIR}/scripts/hubble.sh" fe-port "/tmp/worktree-a") + + [[ "${first}" == "${second}" ]] || fail "port assignment is not stable" + ((first >= 3001 && first <= 3099)) || fail "port ${first} is outside 3001-3099" +} + +test_concurrent_frontend_claim() { + local worktree="${TEST_TMP}/fe-worktree" fake_node="${TEST_TMP}/fake-node" + local first_log="${TEST_TMP}/first-fe.log" first_pid + mkdir -p "${worktree}/hugegraph-hubble/hubble-fe/node_modules" "${fake_node}/bin" + printf '#!/bin/sh\nexit 0\n' > "${fake_node}/bin/node" + printf '#!/bin/sh\nsleep 1\n' > "${fake_node}/bin/yarn" + chmod +x "${fake_node}/bin/node" "${fake_node}/bin/yarn" + + CODEX_HOME="${TEST_TMP}/codex-home" CODEX_SOURCE_TREE_PATH="${TEST_TMP}/source" \ + CODEX_WORKTREE_PATH="${worktree}" HUBBLE_NODE_HOME="${fake_node}" \ + "${CODEX_DIR}/scripts/hubble.sh" fe-start >"${first_log}" 2>&1 & + first_pid=$! + sleep 0.2 + if CODEX_HOME="${TEST_TMP}/codex-home" CODEX_SOURCE_TREE_PATH="${TEST_TMP}/source" \ + CODEX_WORKTREE_PATH="${worktree}" HUBBLE_NODE_HOME="${fake_node}" \ + "${CODEX_DIR}/scripts/hubble.sh" fe-start >/dev/null 2>&1; then + fail "duplicate frontend start unexpectedly succeeded" + fi + wait "${first_pid}" + [[ -z "$(find "${TEST_TMP}/codex-home" -name 'fe-*.owner' -print -quit)" ]] || + fail "frontend owner was not cleaned up" +} + +test_shared_backend_runtime() { + local runtime + runtime=$(CODEX_HOME="${TEST_TMP}/codex-home" \ + CODEX_SOURCE_TREE_PATH="${TEST_TMP}/source" \ + CODEX_WORKTREE_PATH="${TEST_TMP}/worktree-a" \ + "${CODEX_DIR}/scripts/hubble.sh" be-runtime) + assert_contains "${runtime}" "/hubble-be" + [[ "${runtime}" != *"worktree-a"* ]] || + fail "BE runtime is still scoped to a source worktree" +} + +test_compose_command_is_offline_and_layered() { + local server_repo="${TEST_TMP}/server" + local output + mkdir -p "${server_repo}/docker" + : > "${server_repo}/docker/docker-compose.yml" + + output=$(HUGEGRAPH_SERVER_REPO="${server_repo}" \ + "${CODEX_DIR}/scripts/infra.sh" compose-command low) + assert_contains "${output}" "docker-compose.yml" + assert_contains "${output}" "compose.low-memory.yml" + assert_contains "${output}" "--pull never" +} + +test_server_config_patch() { + local conf="${TEST_TMP}/server-conf" + local output + mkdir -p "${conf}" + cat > "${conf}/rest-server.properties" <<'EOF' +batch.max_write_threads=16 +EOF + cat > "${conf}/gremlin-server.yaml" <<'EOF' +other: 1 +EOF + + output=$(HG_SERVER_CONF_DIR="${conf}" \ + "${CODEX_DIR}/infra/patch-server-config.sh" --patch-only 2>&1) + + grep -qx 'batch.max_write_threads=2' "${conf}/rest-server.properties" + grep -qx 'restserver.min_free_memory=16' "${conf}/rest-server.properties" + grep -qx 'other: 1' "${conf}/gremlin-server.yaml" + assert_contains "${output}" "WARN: property 'restserver.min_free_memory'" + assert_contains "${output}" "WARN: optional YAML key 'threadPoolWorker'" + assert_contains "${output}" "WARN: optional YAML key 'gremlinPool'" +} + +test_server_config_patch_requires_config_files() { + local missing_rest="${TEST_TMP}/server-conf-missing-rest" + local missing_yaml="${TEST_TMP}/server-conf-missing-yaml" output + mkdir -p "${missing_rest}" "${missing_yaml}" + printf 'batch.max_write_threads=16\n' > "${missing_yaml}/rest-server.properties" + + if output=$(HG_SERVER_CONF_DIR="${missing_rest}" \ + "${CODEX_DIR}/infra/patch-server-config.sh" --patch-only 2>&1); then + fail "patch unexpectedly accepted missing configuration files" + fi + assert_contains "${output}" "ERROR: missing ${missing_rest}/rest-server.properties" + + if output=$(HG_SERVER_CONF_DIR="${missing_yaml}" \ + "${CODEX_DIR}/infra/patch-server-config.sh" --patch-only 2>&1); then + fail "patch unexpectedly accepted a missing gremlin-server.yaml" + fi + assert_contains "${output}" "ERROR: missing ${missing_yaml}/gremlin-server.yaml" + grep -qx 'batch.max_write_threads=16' "${missing_yaml}/rest-server.properties" +} + +test_environment_toml Review Comment: ‼️ This new contract suite is not wired into any CI job. Repository workflows do not invoke `environment_test.sh`, and `hubble-ci.yml` does not watch `.codex/**`; the current green Hubble job ran because the README changed and never executed these tests. Please add a workflow/step triggered by `.codex/**` that runs this suite, shell/static checks, and both layered `docker compose config` validations, including the BE and infrastructure lifecycle cases above. ########## .codex/scripts/infra.sh: ########## @@ -0,0 +1,191 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail +unset CDPATH + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +CODEX_DIR=$(cd "${SCRIPT_DIR}/.." && pwd) +REPO_ROOT=$(cd "${CODEX_DIR}/.." && pwd) +SOURCE_TREE=${CODEX_SOURCE_TREE_PATH:-${REPO_ROOT}} + +resolve_server_repo() { + if [[ -n "${HUGEGRAPH_SERVER_REPO:-}" ]]; then + echo "${HUGEGRAPH_SERVER_REPO}" + return + fi + local sibling + sibling=$(cd "${SOURCE_TREE}/.." 2>/dev/null && pwd)/server + if [[ -d "${sibling}/docker" ]]; then + echo "${sibling}" + return + fi + echo "ERROR: set HUGEGRAPH_SERVER_REPO or place server beside toolchain" >&2 + exit 1 +} + +compose_files() { + local profile=${1:-low} + local server_repo=$2 + COMPOSE_ARGS=( + -f "${server_repo}/docker/docker-compose.yml" + -f "${CODEX_DIR}/infra/compose.low-memory.yml" + ) + if [[ "${profile}" == "balanced" ]]; then + COMPOSE_ARGS+=(-f "${CODEX_DIR}/infra/compose.balanced.yml") + elif [[ "${profile}" != "low" ]]; then + echo "ERROR: unknown profile ${profile}" >&2 + exit 2 + fi +} + +print_compose_command() { + local profile=${1:-low} server_repo arg + server_repo=$(resolve_server_repo) + [[ -f "${server_repo}/docker/docker-compose.yml" ]] || { + echo "ERROR: missing server Compose file" >&2 + exit 1 + } + compose_files "${profile}" "${server_repo}" + printf 'TOOLCHAIN_CODEX_DIR=%q HUGEGRAPH_VERSION=latest docker compose' "${CODEX_DIR}" + for arg in "${COMPOSE_ARGS[@]}"; do printf ' %q' "${arg}"; done + printf ' up -d --pull never --wait\n' +} + +warn_dirty_server() { + local server_repo=$1 dirty + dirty=$(git -C "${server_repo}" status --short 2>/dev/null || true) + if [[ -n "${dirty}" ]]; then + echo "WARN: server repo has local changes; latest images do not include them:" >&2 + echo "${dirty}" | head -20 >&2 + fi +} + +assert_docker() { + docker info >/dev/null 2>&1 || { + echo "ERROR: Docker is not running" >&2 + exit 1 + } +} + +assert_port_available_for_compose() { + local port=$1 expected_container=$2 listener container + local project='' service='' mapping='' + listener=$(lsof -tiTCP:"${port}" -sTCP:LISTEN 2>/dev/null | head -1 || true) + [[ -z "${listener}" ]] && return + container=$(docker ps --filter "name=^/${expected_container}$" \ + --format '{{.ID}}' | head -1) + if [[ -n "${container}" ]]; then + project=$(docker inspect --format \ + '{{index .Config.Labels "com.docker.compose.project"}}' "${container}") + service=$(docker inspect --format \ + '{{index .Config.Labels "com.docker.compose.service"}}' "${container}") + mapping=$(docker port "${container}" "${port}/tcp" 2>/dev/null || true) + fi + if [[ -z "${container}" || "${project}" != hugegraph-single || \ + "${service}" != "${expected_container#hg-}" || \ + "${mapping}" != *":${port}"* ]]; then + echo "ERROR: port ${port} is owned by non-Compose pid ${listener}" >&2 + exit 1 + fi +} + +hstore_start() { + local profile=${1:-low} server_repo + server_repo=$(resolve_server_repo) + assert_docker + warn_dirty_server "${server_repo}" + assert_port_available_for_compose 8080 hg-server + assert_port_available_for_compose 8520 hg-store + assert_port_available_for_compose 8620 hg-pd + compose_files "${profile}" "${server_repo}" + echo "[infra] starting HStore 1+1+1 (${profile}, local latest images)" + TOOLCHAIN_CODEX_DIR="${CODEX_DIR}" HUGEGRAPH_VERSION=latest \ + docker compose "${COMPOSE_ARGS[@]}" up -d --pull never --wait +} + +infra_status() { + local server_repo + server_repo=$(resolve_server_repo) + assert_docker + compose_files low "${server_repo}" + TOOLCHAIN_CODEX_DIR="${CODEX_DIR}" HUGEGRAPH_VERSION=latest \ + docker compose "${COMPOSE_ARGS[@]}" ps + echo + docker stats --no-stream --format \ + 'table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}' \ + hg-pd hg-store hg-server 2>/dev/null || true + echo + lsof -nP -iTCP:8080 -sTCP:LISTEN 2>/dev/null || true +} + +infra_stop() { + local server_repo + server_repo=$(resolve_server_repo) + assert_docker + compose_files low "${server_repo}" + echo "[infra] stopping Compose services; named volumes are preserved" + TOOLCHAIN_CODEX_DIR="${CODEX_DIR}" HUGEGRAPH_VERSION=latest \ + docker compose "${COMPOSE_ARGS[@]}" down Review Comment: ⚠️ This `down` does not preserve the Server state as documented. `hugegraph/server:latest` declares `VOLUME /hugegraph-server`, but the merged Compose model does not attach a named volume there; removing the container leaves an anonymous volume that the next `up` will not reuse, including the initialization marker and local configuration. Please declare a stable named Server volume (and let reset remove it), or use `docker compose stop` for the preserving operation, with a stop/start persistence test. ########## .codex/infra/compose.low-memory.yml: ########## @@ -0,0 +1,108 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +services: + pd: + pull_policy: never + mem_limit: 512m + cpus: 2.0 + environment: + JAVA_OPTS: >- + -Xms64m -Xmx64m + -XX:ActiveProcessorCount=2 + -XX:MaxMetaspaceSize=96m + -XX:MaxDirectMemorySize=64m + -XX:+UseContainerSupport + -Dlog4j2.asyncLoggerConfigRingBufferSize=4096 + -DAsyncLoggerConfig.RingBufferSize=4096 + -Dthread.pool.grpc.core=8 + -Dthread.pool.grpc.max=32 + -Dthread.pool.grpc.queue=256 + -Djob.interruptableThreadPool.core=1 + -Djob.interruptableThreadPool.max=8 + -Djob.interruptableThreadPool.queue=128 + -Djob.uninterruptibleThreadPool.core=0 + -Djob.uninterruptibleThreadPool.max=4 + -Djob.uninterruptibleThreadPool.queue=64 + -Dpartition.default-shard-count=1 + -Dpartition.store-max-shard-count=12 + + store: + pull_policy: never + mem_limit: 512m + cpus: 2.0 + environment: + JAVA_OPTS: >- + -Xms64m -Xmx64m + -XX:ActiveProcessorCount=2 + -XX:MaxMetaspaceSize=96m + -XX:MaxDirectMemorySize=64m + -XX:+UseContainerSupport + -Dlog4j2.asyncLoggerConfigRingBufferSize=4096 + -DAsyncLoggerConfig.RingBufferSize=4096 + -Drocksdb.total_memory_size=67108864 + -Drocksdb.write_buffer_size=1048576 + -Drocksdb.min_write_buffer_number_to_merge=2 + -Dthread.pool.grpc.core=8 + -Dthread.pool.grpc.max=32 + -Dthread.pool.grpc.queue=256 + -Dthread.pool.scan.core=4 + -Dthread.pool.scan.max=16 + -Dthread.pool.scan.queue=128 + -Djob.interruptableThreadPool.core=2 + -Djob.interruptableThreadPool.max=8 + -Djob.interruptableThreadPool.queue=128 + -Djob.uninterruptibleThreadPool.core=0 + -Djob.uninterruptibleThreadPool.max=4 + -Djob.uninterruptibleThreadPool.queue=64 + -Dquery.push-down.threads=8 + -Dquery.push-down.fetch_batch=1000 + -Dquery.push-down.fetch_timeout=30000 + -Dquery.push-down.memory_limit_count=5000 + -Dquery.push-down.index_size_limit_count=5000 + -Draft.disruptorBufferSize=256 + -Draft.metrics=false + -Draft.maxReplicatorInflightMsgs=16 + -Draft.maxEntriesSize=32 + -Draft.maxBodySize=131072 + + server: + pull_policy: never + mem_limit: 768m + cpus: 4.0 + entrypoint: + - /usr/bin/dumb-init + - "--" + - /opt/toolchain-low-memory/patch-server-config.sh + environment: + PASSWORD: ${HUBBLE_SERVER_PASSWORD:-pa} + JAVA_OPTS: >- + -Xms128m -Xmx192m + -XX:ActiveProcessorCount=4 + -XX:MaxMetaspaceSize=128m + -XX:MaxDirectMemorySize=64m + -XX:+UseContainerSupport + -Dlog4j2.asyncLoggerConfigRingBufferSize=4096 + -DAsyncLoggerConfig.RingBufferSize=4096 + --add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED + --add-modules=jdk.unsupported + --add-exports=java.base/sun.nio.ch=ALL-UNNAMED + volumes: + - type: bind + source: ${TOOLCHAIN_CODEX_DIR}/infra/patch-server-config.sh Review Comment: ⚠️ The shared HStore service persists beyond worktree cleanup, but this bind mount points into the worktree that launched it. Once that temporary checkout is deleted, a container restart/recreation can no longer mount or execute the patch script. Please copy the script to a stable runtime location such as `~/.codex/run/...`, or package it in a stable image/volume, before starting the persistent service. ########## .codex/scripts/infra.sh: ########## @@ -0,0 +1,191 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail +unset CDPATH + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +CODEX_DIR=$(cd "${SCRIPT_DIR}/.." && pwd) +REPO_ROOT=$(cd "${CODEX_DIR}/.." && pwd) +SOURCE_TREE=${CODEX_SOURCE_TREE_PATH:-${REPO_ROOT}} + +resolve_server_repo() { + if [[ -n "${HUGEGRAPH_SERVER_REPO:-}" ]]; then + echo "${HUGEGRAPH_SERVER_REPO}" + return + fi + local sibling + sibling=$(cd "${SOURCE_TREE}/.." 2>/dev/null && pwd)/server + if [[ -d "${sibling}/docker" ]]; then + echo "${sibling}" + return + fi + echo "ERROR: set HUGEGRAPH_SERVER_REPO or place server beside toolchain" >&2 + exit 1 +} + +compose_files() { + local profile=${1:-low} + local server_repo=$2 + COMPOSE_ARGS=( + -f "${server_repo}/docker/docker-compose.yml" + -f "${CODEX_DIR}/infra/compose.low-memory.yml" + ) + if [[ "${profile}" == "balanced" ]]; then + COMPOSE_ARGS+=(-f "${CODEX_DIR}/infra/compose.balanced.yml") + elif [[ "${profile}" != "low" ]]; then + echo "ERROR: unknown profile ${profile}" >&2 + exit 2 + fi +} + +print_compose_command() { + local profile=${1:-low} server_repo arg + server_repo=$(resolve_server_repo) + [[ -f "${server_repo}/docker/docker-compose.yml" ]] || { + echo "ERROR: missing server Compose file" >&2 + exit 1 + } + compose_files "${profile}" "${server_repo}" + printf 'TOOLCHAIN_CODEX_DIR=%q HUGEGRAPH_VERSION=latest docker compose' "${CODEX_DIR}" + for arg in "${COMPOSE_ARGS[@]}"; do printf ' %q' "${arg}"; done + printf ' up -d --pull never --wait\n' +} + +warn_dirty_server() { + local server_repo=$1 dirty + dirty=$(git -C "${server_repo}" status --short 2>/dev/null || true) + if [[ -n "${dirty}" ]]; then + echo "WARN: server repo has local changes; latest images do not include them:" >&2 + echo "${dirty}" | head -20 >&2 + fi +} + +assert_docker() { + docker info >/dev/null 2>&1 || { + echo "ERROR: Docker is not running" >&2 + exit 1 + } +} + +assert_port_available_for_compose() { + local port=$1 expected_container=$2 listener container + local project='' service='' mapping='' + listener=$(lsof -tiTCP:"${port}" -sTCP:LISTEN 2>/dev/null | head -1 || true) + [[ -z "${listener}" ]] && return + container=$(docker ps --filter "name=^/${expected_container}$" \ + --format '{{.ID}}' | head -1) + if [[ -n "${container}" ]]; then + project=$(docker inspect --format \ + '{{index .Config.Labels "com.docker.compose.project"}}' "${container}") + service=$(docker inspect --format \ + '{{index .Config.Labels "com.docker.compose.service"}}' "${container}") + mapping=$(docker port "${container}" "${port}/tcp" 2>/dev/null || true) + fi + if [[ -z "${container}" || "${project}" != hugegraph-single || \ + "${service}" != "${expected_container#hg-}" || \ + "${mapping}" != *":${port}"* ]]; then + echo "ERROR: port ${port} is owned by non-Compose pid ${listener}" >&2 + exit 1 + fi +} + +hstore_start() { + local profile=${1:-low} server_repo + server_repo=$(resolve_server_repo) + assert_docker + warn_dirty_server "${server_repo}" + assert_port_available_for_compose 8080 hg-server + assert_port_available_for_compose 8520 hg-store + assert_port_available_for_compose 8620 hg-pd + compose_files "${profile}" "${server_repo}" + echo "[infra] starting HStore 1+1+1 (${profile}, local latest images)" + TOOLCHAIN_CODEX_DIR="${CODEX_DIR}" HUGEGRAPH_VERSION=latest \ + docker compose "${COMPOSE_ARGS[@]}" up -d --pull never --wait +} + +infra_status() { + local server_repo + server_repo=$(resolve_server_repo) + assert_docker + compose_files low "${server_repo}" + TOOLCHAIN_CODEX_DIR="${CODEX_DIR}" HUGEGRAPH_VERSION=latest \ + docker compose "${COMPOSE_ARGS[@]}" ps + echo + docker stats --no-stream --format \ + 'table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}' \ + hg-pd hg-store hg-server 2>/dev/null || true + echo + lsof -nP -iTCP:8080 -sTCP:LISTEN 2>/dev/null || true +} + +infra_stop() { + local server_repo + server_repo=$(resolve_server_repo) + assert_docker + compose_files low "${server_repo}" + echo "[infra] stopping Compose services; named volumes are preserved" + TOOLCHAIN_CODEX_DIR="${CODEX_DIR}" HUGEGRAPH_VERSION=latest \ + docker compose "${COMPOSE_ARGS[@]}" down +} + +infra_reset() { + local server_repo confirmation=${CONFIRM_INFRA_RESET:-} + server_repo=$(resolve_server_repo) + assert_docker + compose_files low "${server_repo}" + if [[ "${confirmation}" != RESET ]]; then + if [[ ! -t 0 ]]; then + echo "ERROR: destructive reset requires CONFIRM_INFRA_RESET=RESET" >&2 + exit 1 + fi + read -r -p "Type RESET to delete local HStore Compose volumes: " confirmation + fi + [[ "${confirmation}" == RESET ]] || { + echo "[infra] reset cancelled" + return + } + echo "[infra] stopping Compose services and deleting their named volumes" + TOOLCHAIN_CODEX_DIR="${CODEX_DIR}" HUGEGRAPH_VERSION=latest \ + docker compose "${COMPOSE_ARGS[@]}" down --volumes --remove-orphans +} + +infra_pull() { + local server_repo + server_repo=$(resolve_server_repo) + assert_docker + compose_files low "${server_repo}" + echo "[infra] explicitly updating latest images" + TOOLCHAIN_CODEX_DIR="${CODEX_DIR}" HUGEGRAPH_VERSION=latest \ + docker compose "${COMPOSE_ARGS[@]}" pull Review Comment: ⚠️ `Infrastructure Pull` currently skips every image because the low-memory overlay sets `pull_policy: never`. On this head, `docker compose ... --dry-run pull` reports `Skipped` for server/store/PD, while adding `--policy always` reports the expected pulls. Please explicitly override the service policy here (for example, `pull --policy always`) and test that this action really refreshes the images. ########## .codex/infra/compose.low-memory.yml: ########## @@ -0,0 +1,108 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +services: + pd: + pull_policy: never + mem_limit: 512m + cpus: 2.0 + environment: + JAVA_OPTS: >- + -Xms64m -Xmx64m + -XX:ActiveProcessorCount=2 + -XX:MaxMetaspaceSize=96m + -XX:MaxDirectMemorySize=64m + -XX:+UseContainerSupport + -Dlog4j2.asyncLoggerConfigRingBufferSize=4096 + -DAsyncLoggerConfig.RingBufferSize=4096 + -Dthread.pool.grpc.core=8 + -Dthread.pool.grpc.max=32 + -Dthread.pool.grpc.queue=256 + -Djob.interruptableThreadPool.core=1 + -Djob.interruptableThreadPool.max=8 + -Djob.interruptableThreadPool.queue=128 + -Djob.uninterruptibleThreadPool.core=0 + -Djob.uninterruptibleThreadPool.max=4 + -Djob.uninterruptibleThreadPool.queue=64 + -Dpartition.default-shard-count=1 + -Dpartition.store-max-shard-count=12 + + store: + pull_policy: never + mem_limit: 512m + cpus: 2.0 + environment: + JAVA_OPTS: >- + -Xms64m -Xmx64m + -XX:ActiveProcessorCount=2 + -XX:MaxMetaspaceSize=96m + -XX:MaxDirectMemorySize=64m + -XX:+UseContainerSupport + -Dlog4j2.asyncLoggerConfigRingBufferSize=4096 + -DAsyncLoggerConfig.RingBufferSize=4096 + -Drocksdb.total_memory_size=67108864 + -Drocksdb.write_buffer_size=1048576 + -Drocksdb.min_write_buffer_number_to_merge=2 + -Dthread.pool.grpc.core=8 + -Dthread.pool.grpc.max=32 + -Dthread.pool.grpc.queue=256 + -Dthread.pool.scan.core=4 + -Dthread.pool.scan.max=16 + -Dthread.pool.scan.queue=128 + -Djob.interruptableThreadPool.core=2 + -Djob.interruptableThreadPool.max=8 + -Djob.interruptableThreadPool.queue=128 + -Djob.uninterruptibleThreadPool.core=0 + -Djob.uninterruptibleThreadPool.max=4 + -Djob.uninterruptibleThreadPool.queue=64 + -Dquery.push-down.threads=8 + -Dquery.push-down.fetch_batch=1000 + -Dquery.push-down.fetch_timeout=30000 + -Dquery.push-down.memory_limit_count=5000 + -Dquery.push-down.index_size_limit_count=5000 + -Draft.disruptorBufferSize=256 + -Draft.metrics=false + -Draft.maxReplicatorInflightMsgs=16 + -Draft.maxEntriesSize=32 + -Draft.maxBodySize=131072 + + server: + pull_policy: never + mem_limit: 768m + cpus: 4.0 + entrypoint: + - /usr/bin/dumb-init + - "--" + - /opt/toolchain-low-memory/patch-server-config.sh + environment: + PASSWORD: ${HUBBLE_SERVER_PASSWORD:-pa} Review Comment: ⚠️ This known default administrator password is combined with ports published on every host interface. The merged Compose config has no `host_ip` for 8080, 8520, or 8620, so the documented `admin/pa` Server and the Store/PD management endpoints are reachable from the developer's LAN. Please bind the necessary Server port to loopback, avoid publishing Store/PD to the host by default, and require or safely generate a local password. ########## .codex/scripts/hubble.sh: ########## @@ -0,0 +1,473 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail +unset CDPATH + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(cd "${SCRIPT_DIR}/../.." && pwd) +WORKTREE=${CODEX_WORKTREE_PATH:-${REPO_ROOT}} +SOURCE_TREE=${CODEX_SOURCE_TREE_PATH:-${REPO_ROOT}} +NODE_HOME=${HUBBLE_NODE_HOME:-"${HOME}/.nvm/versions/node/v18.20.8"} +CODEX_HOME=${CODEX_HOME:-"${HOME}/.codex"} + +hash_text() { + printf '%s' "$1" | LC_ALL=C shasum -a 256 | awk '{print substr($1, 1, 12)}' +} + +STATE_ROOT="${CODEX_HOME}/run/toolchain-$(hash_text "${SOURCE_TREE}")" +WORKTREE_ID=$(hash_text "${WORKTREE}") +RUNTIME_DIR="${STATE_ROOT}/worktrees/${WORKTREE_ID}" +BE_RUNTIME_DIR="${STATE_ROOT}/hubble-be" + +fe_port() { + local path=${1:-${WORKTREE}} + local checksum + checksum=$(printf '%s' "${path}" | cksum | awk '{print $1}') + echo $((3001 + checksum % 99)) +} + +read_field() { + local file=$1 key=$2 + awk -F= -v key="${key}" '$1 == key {sub(/^[^=]*=/, ""); print; exit}' "${file}" +} + +process_alive() { + local pid=$1 + [[ "${pid}" =~ ^[0-9]+$ ]] && kill -0 "${pid}" 2>/dev/null +} + +process_start_time() { + ps -o lstart= -p "$1" 2>/dev/null | sed -E 's/^[[:space:]]+//;s/[[:space:]]+$//' +} + +process_is_descendant() { + local child=$1 ancestor=$2 parent + while [[ "${child}" =~ ^[0-9]+$ ]] && ((child > 1)); do + [[ "${child}" == "${ancestor}" ]] && return 0 + parent=$(ps -o ppid= -p "${child}" 2>/dev/null | tr -d ' ') + [[ -n "${parent}" && "${parent}" != "${child}" ]] || return 1 + child=${parent} + done + return 1 +} + +owner_identity_matches() { + local file=$1 pid expected_start command + pid=$(read_field "${file}" pid) + expected_start=$(read_field "${file}" start_time) + process_alive "${pid}" || return 1 + [[ -n "${expected_start}" ]] || return 1 + [[ "$(process_start_time "${pid}")" == "${expected_start}" ]] || return 1 + command=$(ps -o command= -p "${pid}" 2>/dev/null || true) + [[ "${command}" == *".codex/scripts/hubble.sh"* ]] +} + +owner_service_matches() { + local file=$1 pid port listener + owner_identity_matches "${file}" || return 1 + pid=$(read_field "${file}" pid) + port=$(read_field "${file}" port) + listener=$(lsof -tiTCP:"${port}" -sTCP:LISTEN 2>/dev/null | head -1 || true) + [[ -z "${listener}" ]] || process_is_descendant "${listener}" "${pid}" +} + +acquire_state_lock() { + local attempt probe + mkdir -p "${STATE_ROOT}" + sleep 5 & + probe=$! + STATE_LOCK_PID=$(ps -o ppid= -p "${probe}" | tr -d ' ') + kill "${probe}" 2>/dev/null || true + wait "${probe}" 2>/dev/null || true + [[ "${STATE_LOCK_PID}" =~ ^[0-9]+$ ]] || { + echo "ERROR: unable to identify owner-lock process" >&2 + return 1 + } + if command -v flock >/dev/null 2>&1; then + STATE_LOCK_KIND=flock + STATE_LOCK_PATH="${STATE_ROOT}/.owner.lock" + exec 9>"${STATE_LOCK_PATH}" + flock -w 12 9 || { + echo "ERROR: timed out waiting for Hubble owner lock" >&2 + return 1 + } + return + fi + if command -v shlock >/dev/null 2>&1; then + STATE_LOCK_KIND=shlock + STATE_LOCK_PATH="${STATE_ROOT}/.owner.lock" + for ((attempt = 0; attempt < 240; attempt++)); do + shlock -f "${STATE_LOCK_PATH}" -p "${STATE_LOCK_PID}" && return + sleep 0.05 + done + echo "ERROR: timed out waiting for Hubble owner lock" >&2 + return 1 + fi + + STATE_LOCK_KIND="mkdir" + STATE_LOCK_PATH="${STATE_ROOT}/.owner-lock" + STATE_LOCK_TOKEN="${STATE_LOCK_PID}-${RANDOM}" + for ((attempt = 0; attempt < 240; attempt++)); do + if mkdir "${STATE_LOCK_PATH}" 2>/dev/null; then + printf '%s\n' "${STATE_LOCK_TOKEN}" > "${STATE_LOCK_PATH}/owner" + return + fi + sleep 0.05 + done + echo "ERROR: owner lock is stuck; remove ${STATE_LOCK_PATH} if no action runs" >&2 + return 1 +} + +release_state_lock() { + local holder + case "${STATE_LOCK_KIND:-}" in + flock) + flock -u 9 2>/dev/null || true + exec 9>&- + ;; + shlock) + holder=$(cat "${STATE_LOCK_PATH}" 2>/dev/null || true) + [[ "${holder}" == "${STATE_LOCK_PID}" ]] && rm -f "${STATE_LOCK_PATH}" + ;; + mkdir) + holder=$(cat "${STATE_LOCK_PATH}/owner" 2>/dev/null || true) + [[ "${holder}" == "${STATE_LOCK_TOKEN}" ]] && rm -rf "${STATE_LOCK_PATH}" + ;; + esac +} + +with_state_lock() ( + acquire_state_lock + trap release_state_lock EXIT + "$@" +) + +new_token() { + printf '%s-%s-%s\n' "$$" "$(date +%s)" "${RANDOM}" +} + +write_owner() { + local file=$1 kind=$2 pid=$3 port=$4 token=$5 temp + mkdir -p "$(dirname "${file}")" + temp="${file}.tmp.${pid}.${RANDOM}" + { + echo "kind=${kind}" + echo "pid=${pid}" + echo "start_time=$(process_start_time "${pid}")" + echo "token=${token}" + echo "port=${port}" + echo "worktree=${WORKTREE}" + echo "branch=$(git -C "${WORKTREE}" branch --show-current 2>/dev/null || true)" + echo "head=$(git -C "${WORKTREE}" rev-parse --short HEAD 2>/dev/null || true)" + } > "${temp}" + mv "${temp}" "${file}" +} + +remove_owner_if_token() { + local file=$1 expected_token=$2 + [[ -f "${file}" ]] || return + if [[ "$(read_field "${file}" token)" == "${expected_token}" ]] && \ + [[ "$(read_field "${file}" pid)" == "$$" ]]; then + rm -f "${file}" + fi +} + +cleanup_claim() { + [[ -n "${SERVICE_OWNER:-}" && -n "${SERVICE_TOKEN:-}" ]] || return + [[ -f "${SERVICE_OWNER}" ]] || return + [[ "$(read_field "${SERVICE_OWNER}" token)" == "${SERVICE_TOKEN}" ]] || return + with_state_lock remove_owner_if_token "${SERVICE_OWNER}" "${SERVICE_TOKEN}" || true +} + +kill_tree() { + local pid=$1 child + while read -r child; do + [[ -n "${child}" ]] && kill_tree "${child}" + done < <(pgrep -P "${pid}" 2>/dev/null || true) + kill -TERM "${pid}" 2>/dev/null || true +} + +live_fe_count() { + local file pid count=0 + shopt -s nullglob + for file in "${STATE_ROOT}"/fe-*.owner; do + pid=$(read_field "${file}" pid) + if owner_service_matches "${file}"; then + count=$((count + 1)) + else + echo "WARN: removing stale FE owner for pid ${pid}" >&2 + rm -f "${file}" + fi + done + echo "${count}" +} + +claim_fe() { + local token=$1 preferred offset port owner pid owner_worktree listener + preferred=$(fe_port) + for ((offset = 0; offset < 99; offset++)); do + port=$((3001 + (preferred - 3001 + offset) % 99)) + owner="${STATE_ROOT}/fe-${port}.owner" + if [[ -f "${owner}" ]]; then + pid=$(read_field "${owner}" pid) + owner_worktree=$(read_field "${owner}" worktree) + if owner_service_matches "${owner}"; then + if [[ "${owner_worktree}" == "${WORKTREE}" ]]; then + echo "ERROR: FE already runs from this worktree on port ${port}" >&2 + return 1 + fi + continue + fi + echo "WARN: removing stale FE owner for pid ${pid}" >&2 + rm -f "${owner}" + fi + listener=$(lsof -tiTCP:"${port}" -sTCP:LISTEN 2>/dev/null | head -1 || true) + [[ -z "${listener}" ]] || continue + write_owner "${owner}" fe "$$" "${port}" "${token}" + echo "${port}" + return + done + echo "ERROR: no free Hubble FE port in 3001-3099" >&2 + return 1 +} + +start_fe() { + local port owner count token + mkdir -p "${RUNTIME_DIR}" + token=$(new_token) + port=$(with_state_lock claim_fe "${token}") + owner="${STATE_ROOT}/fe-${port}.owner" + SERVICE_OWNER=${owner} + SERVICE_TOKEN=${token} + trap cleanup_claim EXIT + + count=$(with_state_lock live_fe_count) + if ((count > 3)); then + echo "WARN: ${count} Hubble frontend servers are already running" >&2 + fi + [[ -x "${NODE_HOME}/bin/node" ]] || { + echo "ERROR: missing Node 18.20.8 at ${NODE_HOME}" >&2 + exit 1 + } + [[ -d "${WORKTREE}/hugegraph-hubble/hubble-fe/node_modules" ]] || { + echo "ERROR: frontend dependencies are missing; run the environment setup" >&2 + exit 1 + } + + export PATH="${NODE_HOME}/bin:${PATH}" + cd "${WORKTREE}/hugegraph-hubble/hubble-fe" + echo "[hubble] FE ${WORKTREE} -> http://127.0.0.1:${port}"; + PORT="${port}" BROWSER=none yarn start +} + +java11_home() { + if [[ -n "${JAVA11_HOME:-}" ]] && [[ -x "${JAVA11_HOME}/bin/java" ]]; then + echo "${JAVA11_HOME}" + elif [[ -x /usr/libexec/java_home ]]; then + /usr/libexec/java_home -v 11 2>/dev/null + elif command -v java >/dev/null 2>&1 && \ + java -version 2>&1 | head -1 | grep -Eq 'version "11\.'; then + cd "$(dirname "$(command -v java)")/.." && pwd + else + echo "ERROR: Java 11 is required; set JAVA11_HOME" >&2 + return 1 + fi +} + +maven_runner() { + local java_home=$1 candidate + for candidate in "${HUBBLE_MVND_BIN:-}" \ + /opt/homebrew/opt/mvnd@1/bin/mvnd \ + /usr/local/opt/mvnd@1/bin/mvnd \ + "$(command -v mvnd 2>/dev/null || true)"; do + if [[ -n "${candidate}" ]] && [[ -x "${candidate}" ]] && \ + JAVA_HOME="${java_home}" "${candidate}" --version >/dev/null 2>&1; then + echo "${candidate}" + return + fi + done + echo mvn +} + +loader_home() { + local candidate + if [[ -n "${HUBBLE_LOADER_HOME:-}" ]]; then + echo "${HUBBLE_LOADER_HOME}" + return + fi + candidate=$(find "${SOURCE_TREE}/hugegraph-loader" -maxdepth 1 -type d \ + -name 'apache-hugegraph-loader-*' | sort | tail -1) + if [[ -z "${candidate}" ]]; then + echo "ERROR: no packaged Loader found; set HUBBLE_LOADER_HOME" >&2 + exit 1 + fi + echo "${candidate}" +} + +claim_be() { + local switch=$1 token=$2 owner="${STATE_ROOT}/be.owner" + local pid owner_worktree listener wait_for + if [[ -f "${owner}" ]]; then + pid=$(read_field "${owner}" pid) + owner_worktree=$(read_field "${owner}" worktree) + if owner_service_matches "${owner}"; then + if [[ "${owner_worktree}" == "${WORKTREE}" ]]; then + echo "[hubble] BE already runs from this worktree (pid ${pid})" >&2 + return 10 + fi + if [[ "${switch}" != true ]]; then + echo "ERROR: BE is owned by ${owner_worktree} (pid ${pid})" >&2 + echo "Use the 'Hubble BE Switch' action to replace it." >&2 + return 1 + fi + rm -f "${owner}" Review Comment: ‼️ The switch path drops ownership before the service has actually stopped. `kill_tree()` only sends TERM; this loop waits for the wrapper shell PID, not the Java listener/descendants, so a slow graceful shutdown leaves an unowned listener and the subsequent port check rejects the switch. `stop_current()` has the same ownership-loss pattern without any wait. Please retain the owner until the captured process tree and port have exited (with a bounded timeout and non-zero failure), and add a delayed-TERM lifecycle test. ########## .codex/scripts/hubble.sh: ########## @@ -0,0 +1,473 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail +unset CDPATH + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(cd "${SCRIPT_DIR}/../.." && pwd) +WORKTREE=${CODEX_WORKTREE_PATH:-${REPO_ROOT}} +SOURCE_TREE=${CODEX_SOURCE_TREE_PATH:-${REPO_ROOT}} +NODE_HOME=${HUBBLE_NODE_HOME:-"${HOME}/.nvm/versions/node/v18.20.8"} +CODEX_HOME=${CODEX_HOME:-"${HOME}/.codex"} + +hash_text() { + printf '%s' "$1" | LC_ALL=C shasum -a 256 | awk '{print substr($1, 1, 12)}' +} + +STATE_ROOT="${CODEX_HOME}/run/toolchain-$(hash_text "${SOURCE_TREE}")" +WORKTREE_ID=$(hash_text "${WORKTREE}") +RUNTIME_DIR="${STATE_ROOT}/worktrees/${WORKTREE_ID}" +BE_RUNTIME_DIR="${STATE_ROOT}/hubble-be" + +fe_port() { + local path=${1:-${WORKTREE}} + local checksum + checksum=$(printf '%s' "${path}" | cksum | awk '{print $1}') + echo $((3001 + checksum % 99)) +} + +read_field() { + local file=$1 key=$2 + awk -F= -v key="${key}" '$1 == key {sub(/^[^=]*=/, ""); print; exit}' "${file}" +} + +process_alive() { + local pid=$1 + [[ "${pid}" =~ ^[0-9]+$ ]] && kill -0 "${pid}" 2>/dev/null +} + +process_start_time() { + ps -o lstart= -p "$1" 2>/dev/null | sed -E 's/^[[:space:]]+//;s/[[:space:]]+$//' +} + +process_is_descendant() { + local child=$1 ancestor=$2 parent + while [[ "${child}" =~ ^[0-9]+$ ]] && ((child > 1)); do + [[ "${child}" == "${ancestor}" ]] && return 0 + parent=$(ps -o ppid= -p "${child}" 2>/dev/null | tr -d ' ') + [[ -n "${parent}" && "${parent}" != "${child}" ]] || return 1 + child=${parent} + done + return 1 +} + +owner_identity_matches() { + local file=$1 pid expected_start command + pid=$(read_field "${file}" pid) + expected_start=$(read_field "${file}" start_time) + process_alive "${pid}" || return 1 + [[ -n "${expected_start}" ]] || return 1 + [[ "$(process_start_time "${pid}")" == "${expected_start}" ]] || return 1 + command=$(ps -o command= -p "${pid}" 2>/dev/null || true) + [[ "${command}" == *".codex/scripts/hubble.sh"* ]] +} + +owner_service_matches() { + local file=$1 pid port listener + owner_identity_matches "${file}" || return 1 + pid=$(read_field "${file}" pid) + port=$(read_field "${file}" port) + listener=$(lsof -tiTCP:"${port}" -sTCP:LISTEN 2>/dev/null | head -1 || true) Review Comment: ⚠️ Missing `lsof` is silently treated as 'no listener'; similarly, `kill_tree()` swallows a missing `pgrep`. On a host without these undeclared tools, the script can claim an occupied port or stop only the wrapper and orphan Java/Node children. Please validate these required commands during setup/start with an actionable error, or provide reliable portable fallbacks instead of converting command-not-found into an empty result. ########## .codex/tests/environment_test.sh: ########## @@ -0,0 +1,185 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail +unset CDPATH + +CODEX_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +REPO_ROOT=$(cd "${CODEX_DIR}/.." && pwd) +TEST_TMP=$(mktemp -d "${TMPDIR:-/tmp}/toolchain-env-test.XXXXXX") +TEST_TMP=$(cd "${TEST_TMP}" && pwd) +trap 'rm -rf "${TEST_TMP}"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +assert_contains() { + local actual=$1 + local expected=$2 + [[ "${actual}" == *"${expected}"* ]] || + fail "expected '${actual}' to contain '${expected}'" +} + +test_environment_toml() { + python3 - "${CODEX_DIR}/environments/environment.toml" <<'PY' +import pathlib +import sys +import tomllib Review Comment: ⚠️ The advertised `bash .codex/tests/environment_test.sh` command is not runnable with the target Mac's default Python: `/usr/bin/python3` 3.9.6 fails here with `ModuleNotFoundError: No module named 'tomllib'`, while setup neither provisions nor checks Python 3.11+. Please use a compatible parser/fallback, or explicitly provision and validate Python 3.11+ in setup, documentation, and CI. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
