This adds a minimal RDMA CM selftest suite that captures observability baselines and runs trace, counter-delta, and fault-injection-oriented checks, plus a review-loop helper for repeated validation rounds.
Signed-off-by: Chenguang Zhao <[email protected]> --- v3: TARGET += rdma is already present, remove it as suggested by Yanjun. v2: https://lore.kernel.org/all/[email protected]/ v1: https://lore.kernel.org/all/[email protected]/ --- tools/testing/selftests/rdma/Makefile | 10 ++ tools/testing/selftests/rdma/config | 6 + .../selftests/rdma/rdma_cm_baseline.sh | 58 ++++++++ .../selftests/rdma/rdma_cm_counter_delta.sh | 72 ++++++++++ .../selftests/rdma/rdma_cm_fault_injection.sh | 95 +++++++++++++ .../selftests/rdma/rdma_cm_review_loop.sh | 35 +++++ .../selftests/rdma/rdma_cm_trace_sequence.sh | 83 ++++++++++++ tools/testing/selftests/rdma/rdma_common.sh | 126 ++++++++++++++++++ 8 files changed, 485 insertions(+) create mode 100755 tools/testing/selftests/rdma/rdma_cm_baseline.sh create mode 100755 tools/testing/selftests/rdma/rdma_cm_counter_delta.sh create mode 100755 tools/testing/selftests/rdma/rdma_cm_fault_injection.sh create mode 100755 tools/testing/selftests/rdma/rdma_cm_review_loop.sh create mode 100755 tools/testing/selftests/rdma/rdma_cm_trace_sequence.sh create mode 100755 tools/testing/selftests/rdma/rdma_common.sh diff --git a/tools/testing/selftests/rdma/Makefile b/tools/testing/selftests/rdma/Makefile index 7dd7cba7a73c..04c52db4b9d9 100644 --- a/tools/testing/selftests/rdma/Makefile +++ b/tools/testing/selftests/rdma/Makefile @@ -4,4 +4,14 @@ TEST_PROGS := rxe_rping_between_netns.sh \ rxe_socket_with_netns.sh \ rxe_test_NETDEV_UNREGISTER.sh +TEST_PROGS += \ + rdma_cm_baseline.sh \ + rdma_cm_trace_sequence.sh \ + rdma_cm_counter_delta.sh \ + rdma_cm_fault_injection.sh + +TEST_FILES += \ + rdma_common.sh \ + rdma_cm_review_loop.sh + include ../lib.mk diff --git a/tools/testing/selftests/rdma/config b/tools/testing/selftests/rdma/config index 4ffb814e253b..e22141838c19 100644 --- a/tools/testing/selftests/rdma/config +++ b/tools/testing/selftests/rdma/config @@ -1,3 +1,9 @@ CONFIG_TUN CONFIG_VETH CONFIG_RDMA_RXE +CONFIG_DEBUG_KERNEL +CONFIG_FAULT_INJECTION +CONFIG_SYSFS +CONFIG_DEBUG_FS +CONFIG_FAULT_INJECTION_DEBUG_FS +CONFIG_FAILSLAB diff --git a/tools/testing/selftests/rdma/rdma_cm_baseline.sh b/tools/testing/selftests/rdma/rdma_cm_baseline.sh new file mode 100755 index 000000000000..b0d8b3e46470 --- /dev/null +++ b/tools/testing/selftests/rdma/rdma_cm_baseline.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/rdma_common.sh" + +require_root +require_cmd date +require_cmd uname + +trace_dir="$(tracefs_dir || true)" +counter_root="$(find_cm_counter_root || true)" +out_dir="/tmp/rdma_cm_baseline.$(date +%s)" +dmesg_lines=400 +dmesg_pattern="ib_cm|infiniband|rdma|roce|mlx|hns_roce|irdma|siw|rxe" + +mkdir -p "${out_dir}" + +log_info "writing baseline to ${out_dir}" + +{ + echo "timestamp=$(date -u +%FT%TZ)" + echo "kernel=$(uname -r)" + echo "hostname=$(uname -n)" + echo "dmesg_lines=${dmesg_lines}" + echo "dmesg_pattern=${dmesg_pattern}" +} >"${out_dir}/env.txt" + +if [[ -n "${trace_dir}" && -d "${trace_dir}/events/ib_cma" ]]; then + find "${trace_dir}/events/ib_cma" -maxdepth 2 -name enable -print \ + >"${out_dir}/trace_events.list" 2>/dev/null || true +else + log_warn "tracefs or ib_cma trace events are unavailable" +fi + +if [[ -n "${counter_root}" ]]; then + { + echo "counter_root=${counter_root}" + for group in "${RDMA_COUNTER_GROUPS[@]}"; do + for attr in "${RDMA_COUNTER_ATTRS[@]}"; do + value="$(read_cm_counter "${counter_root}" "${group}" "${attr}")" + echo "${group}.${attr}=${value}" + done + done + } >"${out_dir}/cm_counters.before" +else + log_warn "cm counters are unavailable under /sys/class/infiniband" +fi + +if command -v dmesg >/dev/null 2>&1; then + dmesg | tail -n "${dmesg_lines}" | grep -E "${dmesg_pattern}" \ + >"${out_dir}/dmesg.rdma.tail" || true +fi + +log_info "baseline collection completed" +exit 0 diff --git a/tools/testing/selftests/rdma/rdma_cm_counter_delta.sh b/tools/testing/selftests/rdma/rdma_cm_counter_delta.sh new file mode 100755 index 000000000000..060adf9fe78a --- /dev/null +++ b/tools/testing/selftests/rdma/rdma_cm_counter_delta.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/rdma_common.sh" + +require_root +counter_root="$(find_cm_counter_root || true)" +counter_wait_sec=2 + +if [[ -z "${counter_root}" ]]; then + log_warn "cm counters are unavailable under /sys/class/infiniband" + exit "${ksft_skip}" +fi + +declare -A before after + +for group in "${RDMA_COUNTER_GROUPS[@]}"; do + for attr in "${RDMA_COUNTER_ATTRS[@]}"; do + key="${group}.${attr}" + before["${key}"]="$(read_cm_counter "${counter_root}" "${group}" "${attr}")" + done +done + +if [[ "${counter_wait_sec}" != "0" ]]; then + log_info "waiting ${counter_wait_sec}s before workload" + sleep "${counter_wait_sec}" +fi + +workload_rc=0 +run_workload || workload_rc=$? +if [[ "${workload_rc}" -eq "${ksft_skip}" ]]; then + exit "${ksft_skip}" +fi +if [[ "${workload_rc}" -ne 0 ]]; then + log_err "workload failed with rc=${workload_rc}" + exit "${workload_rc}" +fi + +for group in "${RDMA_COUNTER_GROUPS[@]}"; do + for attr in "${RDMA_COUNTER_ATTRS[@]}"; do + key="${group}.${attr}" + after["${key}"]="$(read_cm_counter "${counter_root}" "${group}" "${attr}")" + delta=$((after["${key}"] - before["${key}"])) + echo "${key}.delta=${delta}" + if ((delta < 0)); then + log_err "counter regressed: ${key}" + exit 1 + fi + done +done + +dup_limit=10 +retry_limit=10 + +for attr in "${RDMA_COUNTER_ATTRS[@]}"; do + dup_delta=$((after["cm_rx_duplicates.${attr}"] - before["cm_rx_duplicates.${attr}"])) + retry_delta=$((after["cm_tx_retries.${attr}"] - before["cm_tx_retries.${attr}"])) + + if ((dup_delta > dup_limit)); then + log_err "duplicate counter exceeds limit: ${attr}=${dup_delta}" + exit 1 + fi + if ((retry_delta > retry_limit)); then + log_err "retry counter exceeds limit: ${attr}=${retry_delta}" + exit 1 + fi +done + +exit 0 diff --git a/tools/testing/selftests/rdma/rdma_cm_fault_injection.sh b/tools/testing/selftests/rdma/rdma_cm_fault_injection.sh new file mode 100755 index 000000000000..0202ee901386 --- /dev/null +++ b/tools/testing/selftests/rdma/rdma_cm_fault_injection.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/rdma_common.sh" + +require_root + +debugfs_fail="/sys/kernel/debug/failslab" +recovery_wait_sec=2 +if [[ ! -d "${debugfs_fail}" ]]; then + log_warn "failslab is unavailable: ${debugfs_fail}" + exit "${ksft_skip}" +fi + +for knob in probability interval times task-filter; do + if [[ ! -f "${debugfs_fail}/${knob}" ]]; then + log_warn "failslab knob missing: ${knob}" + exit "${ksft_skip}" + fi +done + +orig_probability="$(cat "${debugfs_fail}/probability")" +orig_interval="$(cat "${debugfs_fail}/interval")" +orig_times="$(cat "${debugfs_fail}/times")" +orig_task_filter="$(cat "${debugfs_fail}/task-filter")" + +restore_knobs() +{ + echo "${orig_probability}" >"${debugfs_fail}/probability" || true + echo "${orig_interval}" >"${debugfs_fail}/interval" || true + echo "${orig_times}" >"${debugfs_fail}/times" || true + echo "${orig_task_filter}" >"${debugfs_fail}/task-filter" || true +} + +trap restore_knobs EXIT + +log_failslab_state() +{ + local state="$1" + local task_filter probability interval times + + task_filter="$(cat "${debugfs_fail}/task-filter")" + probability="$(cat "${debugfs_fail}/probability")" + interval="$(cat "${debugfs_fail}/interval")" + times="$(cat "${debugfs_fail}/times")" + + log_info "failslab ${state}: task-filter=${task_filter} probability=${probability}" + log_info "failslab ${state}: interval=${interval} times=${times}" +} + +echo 1 >"${debugfs_fail}/task-filter" +echo 1 >"${debugfs_fail}/probability" +echo 100 >"${debugfs_fail}/interval" +echo 1 >"${debugfs_fail}/times" +log_failslab_state "enabled" + +if [[ -z "${CM_WORKLOAD_CMD:-}" && -n "${CM_VALIDATE_RECOVERY_CMD:-}" ]]; then + CM_WORKLOAD_CMD="${CM_VALIDATE_RECOVERY_CMD}" + log_warn "CM_WORKLOAD_CMD is not set; fallback to CM_VALIDATE_RECOVERY_CMD" +fi + +injected_rc=0 +run_workload || injected_rc=$? +if [[ "${injected_rc}" -eq "${ksft_skip}" ]]; then + exit "${ksft_skip}" +fi +log_info "workload rc under injection=${injected_rc}" + +echo 0 >"${debugfs_fail}/probability" +echo 0 >"${debugfs_fail}/times" +echo 0 >"${debugfs_fail}/task-filter" +log_failslab_state "disabled" + +recovery_cmd="${CM_VALIDATE_RECOVERY_CMD:-${CM_WORKLOAD_CMD:-}}" +if [[ -z "${recovery_cmd}" ]]; then + log_warn "CM_VALIDATE_RECOVERY_CMD and CM_WORKLOAD_CMD are both unset" + exit "${ksft_skip}" +fi + +if [[ "${recovery_wait_sec}" != "0" ]]; then + log_info "waiting ${recovery_wait_sec}s before recovery workload" + sleep "${recovery_wait_sec}" +fi + +log_info "running recovery workload: ${recovery_cmd}" +if ! bash -c "${recovery_cmd}"; then + log_err "recovery workload failed after disabling fault injection" + log_err "hint: ensure remote server is restarted and listening for a second connection" + exit 1 +fi + +exit 0 diff --git a/tools/testing/selftests/rdma/rdma_cm_review_loop.sh b/tools/testing/selftests/rdma/rdma_cm_review_loop.sh new file mode 100755 index 000000000000..c156090b17e3 --- /dev/null +++ b/tools/testing/selftests/rdma/rdma_cm_review_loop.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +cd "${SCRIPT_DIR}" + +declare -A rc + +run_step() +{ + local name="$1" + local cmd="$2" + + echo "==== ${name} ====" + if bash -c "${cmd}"; then + rc["${name}"]=0 + else + rc["${name}"]=$? + fi + echo "==== ${name} rc=${rc["${name}"]} ====" +} + +run_step baseline "./rdma_cm_baseline.sh" +run_step trace "./rdma_cm_trace_sequence.sh" +run_step counters "./rdma_cm_counter_delta.sh" +run_step fault_injection "./rdma_cm_fault_injection.sh" + +echo "==== summary ====" +for name in baseline trace counters fault_injection; do + echo "${name}=${rc["${name}"]}" +done + +exit 0 diff --git a/tools/testing/selftests/rdma/rdma_cm_trace_sequence.sh b/tools/testing/selftests/rdma/rdma_cm_trace_sequence.sh new file mode 100755 index 000000000000..7e68289345e8 --- /dev/null +++ b/tools/testing/selftests/rdma/rdma_cm_trace_sequence.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/rdma_common.sh" + +require_root +require_cmd bash +require_cmd grep + +trace_dir="$(tracefs_dir || true)" +if [[ -z "${trace_dir}" ]]; then + log_warn "tracefs is unavailable" + exit "${ksft_skip}" +fi + +if [[ ! -d "${trace_dir}/events/ib_cma" ]]; then + log_warn "ib_cma trace events are unavailable" + exit "${ksft_skip}" +fi + +workload_rc=0 + +cleanup_trace() +{ + local event + + for event in icm_send_req icm_send_rep icm_send_rtu icm_recv_unknown_attr; do + [[ -f "${trace_dir}/events/ib_cma/${event}/enable" ]] && \ + echo 0 >"${trace_dir}/events/ib_cma/${event}/enable" + done + [[ -f "${trace_dir}/events/ib_cma/enable" ]] && echo 0 >"${trace_dir}/events/ib_cma/enable" + echo 0 >"${trace_dir}/tracing_on" +} + +trap cleanup_trace EXIT + +echo 0 >"${trace_dir}/tracing_on" +echo >"${trace_dir}/trace" +echo 1 >"${trace_dir}/events/ib_cma/enable" + +for event in icm_send_req icm_send_rep icm_send_rtu; do + if [[ -f "${trace_dir}/events/ib_cma/${event}/enable" ]]; then + echo 1 >"${trace_dir}/events/ib_cma/${event}/enable" + fi +done + +echo 1 >"${trace_dir}/tracing_on" +run_workload || workload_rc=$? +echo 0 >"${trace_dir}/tracing_on" + +if [[ "${workload_rc}" -eq "${ksft_skip}" ]]; then + exit "${ksft_skip}" +fi + +trace_log="/tmp/rdma_cm_trace.$(date +%s).log" +cat "${trace_dir}/trace" >"${trace_log}" +log_info "captured trace at ${trace_log}" + +if ! grep -Eq "icm_send_(req|rep|rtu)" "${trace_log}"; then + log_err "missing CM send trace events (req/rep/rtu)" + exit 1 +fi + +err_lines="$(grep "icm_.*_err" "${trace_log}" || true)" +if [[ -n "${err_lines}" ]]; then + # DREP send failure while already in TIMEWAIT is a common teardown + # race and is tolerated for this smoke-style validation script. + untolerated_err_lines="$( + printf '%s\n' "${err_lines}" | \ + grep -Ev "icm_send_drep_err: .*state=TIMEWAIT" || true + )" + if [[ -n "${untolerated_err_lines}" ]]; then + log_err "error trace event detected in ib_cma path" + printf '%s\n' "${untolerated_err_lines}" >&2 + exit 1 + fi + log_warn "only tolerated TIMEWAIT drep errors observed" +fi + +exit 0 diff --git a/tools/testing/selftests/rdma/rdma_common.sh b/tools/testing/selftests/rdma/rdma_common.sh new file mode 100755 index 000000000000..ee3d8b0d86b2 --- /dev/null +++ b/tools/testing/selftests/rdma/rdma_common.sh @@ -0,0 +1,126 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +ksft_skip=4 +RET=0 + +RDMA_COUNTER_GROUPS=( + cm_tx_msgs + cm_tx_retries + cm_rx_msgs + cm_rx_duplicates +) + +RDMA_COUNTER_ATTRS=( + req + mra + rej + rep + rtu + dreq + drep + sidr_req + sidr_rep + lap + apr +) + +log_info() +{ + echo "INFO: $*" +} + +log_warn() +{ + echo "WARN: $*" >&2 +} + +log_err() +{ + echo "ERROR: $*" >&2 +} + +require_root() +{ + if [[ "$(id -u)" -ne 0 ]]; then + log_warn "this test requires root privileges" + exit "${ksft_skip}" + fi +} + +require_cmd() +{ + local cmd="$1" + + command -v "${cmd}" >/dev/null 2>&1 || { + log_warn "missing required command: ${cmd}" + exit "${ksft_skip}" + } +} + +tracefs_dir() +{ + if [[ -d /sys/kernel/tracing ]]; then + echo /sys/kernel/tracing + elif [[ -d /sys/kernel/debug/tracing ]]; then + echo /sys/kernel/debug/tracing + else + return 1 + fi +} + +find_cm_counter_root() +{ + local base + local port + local candidate + + for base in /sys/class/infiniband/*; do + [[ -d "${base}" ]] || continue + + for port in "${base}"/ports/*; do + [[ -d "${port}" ]] || continue + # RoCE / newer sysfs: cm_* groups live directly under ports/<N>/ + if [[ -d "${port}/cm_tx_msgs" ]]; then + echo "${port}" + return 0 + fi + # Legacy layout: under counters/ or hw_counters/ + for candidate in "${port}/counters" "${port}/hw_counters"; do + [[ -d "${candidate}/cm_tx_msgs" ]] || continue + echo "${candidate}" + return 0 + done + done + done + + return 1 +} + +read_cm_counter() +{ + local root="$1" + local group="$2" + local attr="$3" + local path="${root}/${group}/${attr}" + + if [[ -f "${path}" ]]; then + cat "${path}" 2>/dev/null + else + echo 0 + fi +} + +run_workload() +{ + local cmd="${CM_WORKLOAD_CMD:-}" + + if [[ -z "${cmd}" ]]; then + log_warn "CM_WORKLOAD_CMD is not set" + return "${ksft_skip}" + fi + + log_info "running workload: ${cmd}" + bash -c "${cmd}" +} + -- 2.25.1
