This is an automated email from the ASF dual-hosted git repository.
sergehuber pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new 2a2f6cf9e UNOMI-957: Add cross-platform memory sampling for
integration test runs (#782)
2a2f6cf9e is described below
commit 2a2f6cf9e5dfa3be064b67b5d22c4d139c883c37
Author: Serge Huber <[email protected]>
AuthorDate: Sat Jun 27 21:56:15 2026 +0200
UNOMI-957: Add cross-platform memory sampling for integration test runs
(#782)
Merge memory sampling improvement
---
.github/workflows/unomi-ci-build-tests.yml | 10 +
build.sh | 43 ++
itests/README.md | 56 +-
itests/archive-it-run.sh | 36 +-
itests/lib/it-run-context.sh | 6 +
itests/lib/it-run-memory.sh | 651 +++++++++++++++++++++
itests/lib/it-run.sh | 200 ++++++-
itests/sample-it-memory.sh | 229 ++++++++
.../impl/cluster/ClusterServiceImplTest.java | 23 +-
.../impl/scheduler/SchedulerServiceImplTest.java | 16 +-
10 files changed, 1249 insertions(+), 21 deletions(-)
diff --git a/.github/workflows/unomi-ci-build-tests.yml
b/.github/workflows/unomi-ci-build-tests.yml
index af90f2747..2b7e190dd 100644
--- a/.github/workflows/unomi-ci-build-tests.yml
+++ b/.github/workflows/unomi-ci-build-tests.yml
@@ -70,6 +70,16 @@ jobs:
else
./build.sh --ci --integration-tests
fi
+ - name: Archive IT memory metrics
+ uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: it-memory-metrics-jdk17-${{ matrix.search-engine }}-${{
github.run_number }}
+ path: |
+ itests/target/memory-samples.tsv
+ itests/target/memory-summary.txt
+ itests/target/memory-sampler.log
+ if-no-files-found: ignore
- name: Archive code coverage logs
uses: actions/upload-artifact@v4
if: false # UNOMI-746 Reactivate if necessary
diff --git a/build.sh b/build.sh
index a88ce8a0f..c6eaa7837 100755
--- a/build.sh
+++ b/build.sh
@@ -272,6 +272,8 @@ IT_DEBUG_PORT=5006
IT_DEBUG_SUSPEND=false
SKIP_MIGRATION_TESTS=false
KEEP_CONTAINER=false
+IT_MEMORY_SAMPLER=true
+IT_MEMORY_INTERVAL=30
JAVADOC=false
LOG_FILE=""
LOG_FILE_ONLY=false
@@ -315,6 +317,8 @@ EOF
echo -e " ${CYAN}--it-debug-suspend${NC} Suspend integration
test until debugger connects"
echo -e " ${CYAN}--skip-migration-tests${NC} Skip
migration-related tests"
echo -e " ${CYAN}--keep-container${NC} Keep search engine
container running after tests (for post-failure inspection)"
+ echo -e " ${CYAN}--no-memory-sampler${NC} Disable JVM/system
memory sampling during integration tests"
+ echo -e " ${CYAN}--memory-interval SEC${NC} Memory sample interval
in seconds (default: 30)"
echo -e " ${CYAN}--javadoc${NC} Build and validate
Javadoc after install (fails on doclint errors)"
echo -e " ${CYAN}--ci${NC} CI mode: no Karaf,
no Maven build cache, non-interactive, includes Javadoc"
echo -e " ${CYAN}--log-file PATH${NC} Tee all output to
PATH (console + file)"
@@ -354,6 +358,8 @@ EOF
echo " --it-debug-suspend Suspend integration test until
debugger connects"
echo " --skip-migration-tests Skip migration-related tests"
echo " --keep-container Keep search engine container running
after tests (for post-failure inspection)"
+ echo " --no-memory-sampler Disable JVM/system memory sampling
during integration tests"
+ echo " --memory-interval SEC Memory sample interval in seconds
(default: 30)"
echo " --javadoc Build and validate Javadoc after
install (fails on doclint errors)"
echo " --ci CI mode: no Karaf, no Maven build
cache, non-interactive, includes Javadoc"
echo " --log-file PATH Tee all output to PATH (console +
file)"
@@ -505,6 +511,13 @@ while [ "$1" != "" ]; do
--keep-container)
KEEP_CONTAINER=true
;;
+ --no-memory-sampler)
+ IT_MEMORY_SAMPLER=false
+ ;;
+ --memory-interval)
+ shift
+ IT_MEMORY_INTERVAL="$1"
+ ;;
--javadoc)
JAVADOC=true
;;
@@ -1037,6 +1050,8 @@ write_it_run_trace_start() {
echo "it.debug.suspend=$IT_DEBUG_SUSPEND"
echo "skip.migration.tests=$SKIP_MIGRATION_TESTS"
echo "it.keep.container=$KEEP_CONTAINER"
+ echo "it.memory.sampler=$IT_MEMORY_SAMPLER"
+ echo "it.memory.interval=$IT_MEMORY_INTERVAL"
echo "maven.debug=$MAVEN_DEBUG"
echo "maven.offline=$MAVEN_OFFLINE"
echo "maven.quiet=$MAVEN_QUIET"
@@ -1048,6 +1063,31 @@ write_it_run_trace_start() {
} > "$trace_file"
}
+
+start_it_memory_sampler() {
+ local sampler="$DIRNAME/itests/sample-it-memory.sh"
+ if [ "$IT_MEMORY_SAMPLER" != true ] || [ ! -f "$sampler" ]; then
+ return 0
+ fi
+ bash "$sampler" start --target-dir "$DIRNAME/itests/target" --interval
"$IT_MEMORY_INTERVAL" || \
+ print_status "warning" "Could not start IT memory sampler"
+}
+
+write_it_run_operator_note() {
+ local sampler="$DIRNAME/itests/sample-it-memory.sh"
+ if [ -f "$sampler" ]; then
+ bash "$sampler" operator-note --target-dir "$DIRNAME/itests/target"
2>/dev/null || true
+ fi
+}
+
+stop_it_memory_sampler() {
+ local sampler="$DIRNAME/itests/sample-it-memory.sh"
+ if [ ! -f "$sampler" ]; then
+ return 0
+ fi
+ bash "$sampler" stop --target-dir "$DIRNAME/itests/target" 2>/dev/null ||
true
+}
+
finalize_it_run_trace() {
local exit_code="$1"
local trace_file="$DIRNAME/itests/target/it-run-trace.properties"
@@ -1074,6 +1114,7 @@ $MVN_CMD clean $MVN_OPTS || {
if [ "$RUN_INTEGRATION_TESTS" = true ]; then
write_it_run_trace_start
+ start_it_memory_sampler
fi
print_progress $((++current_step)) $total_steps "Compiling and installing
artifacts..."
@@ -1085,7 +1126,9 @@ fi
INSTALL_EXIT=0
$MVN_CMD install $MVN_OPTS || INSTALL_EXIT=$?
if [ "$RUN_INTEGRATION_TESTS" = true ]; then
+ stop_it_memory_sampler
finalize_it_run_trace "$INSTALL_EXIT"
+ write_it_run_operator_note
fi
if [ "$INSTALL_EXIT" -ne 0 ]; then
print_status "error" "Maven install failed"
diff --git a/itests/README.md b/itests/README.md
index b2500f8fb..e3929d611 100644
--- a/itests/README.md
+++ b/itests/README.md
@@ -45,8 +45,60 @@ run under `itests/target/exam/` with a UUID directory name.
| `kt.sh` | Live Karaf inspection during a run (log, tail, grep, debug) |
| `archive-it-run.sh` | Capture IT run artifacts before the next build wipes
them |
| `compare-it-runs.sh` | Diff multiple captures to classify flaky vs
systematic failures |
+| `sample-it-memory.sh` | Memory sampling, summarize, operator note, and
cross-platform verify |
| `jacoco-report.sh` | Generate a JaCoCo coverage report after a run |
+
+---
+
+## Memory sizing analysis
+
+`./build.sh --integration-tests` starts a background memory sampler
automatically.
+Results land in `itests/target/`:
+
+| File | Purpose |
+|------|---------|
+| `memory-samples.tsv` | Time-series samples (Karaf heap, search heap, Docker
RSS, swap) |
+| `memory-summary.txt` | Peak usage, headroom %, peak Karaf GCT (seconds),
swap warnings |
+| `memory-sampler.log` | Sampler diagnostics |
+
+Disable sampling with `--no-memory-sampler`, or change the interval with
+`--memory-interval 60` (default: 30s).
+
+Each sample uses one header-aware `jstat -gc` attach (cached `MaxHeapSize` per
Karaf PID,
+with `jinfo`/`jcmd`/trace fallbacks), a filtered ES `/_nodes/stats/jvm`
request, and
+`docker stats` on the IT container only (`itests-elasticsearch` or
`itests-opensearch`).
+System metrics use `vm_stat`/`sysctl` on macOS and `free`/`/proc/loadavg` on
Linux.
+
+Swap warnings fire only when available RAM drops below 2 GB during the run or
swap
+grows by more than 256 MB (avoids false positives from baseline macOS swap).
+
+Verify locally:
+
+```bash
+cd itests
+./sample-it-memory.sh verify
+```
+
+Manual usage:
+
+```bash
+./itests/sample-it-memory.sh start --interval 30
+./itests/sample-it-memory.sh stop # writes memory-summary.txt
+./itests/sample-it-memory.sh summarize
+./itests/sample-it-memory.sh operator-note --print-only
+```
+
+After a run, `build.sh` writes `itests/target/it-run-operator-note.txt` with
outcome,
+heap config, test counts, and memory peaks. `archive-it-run.sh` uses that file
as the
+default operator note when you omit `-m` (override with `-m` or
`--message-file` as before).
+
+The archive also includes memory files and adds peak metrics to
`run-summary.properties`.
+On GitHub Actions, download the `it-memory-metrics-*` artifact from the
workflow run.
+
+Compare heap configurations across runs using `archives/runs-index.tsv`
(configured
+heaps + observed peaks from `memory.peak.*` fields in each capture).
+
---
## Writing Integration Tests
@@ -325,8 +377,8 @@ wipes `itests/target/`:
```bash
cd itests
-./archive-it-run.sh
-./archive-it-run.sh -m "Heavy swap, 2 failures in GraphQLListIT" # with an
operator note
+./archive-it-run.sh # uses auto-generated
operator note
+./archive-it-run.sh -m "Heavy swap, 2 failures in GraphQLListIT" # override
auto note
./archive-it-run.sh --full-karaf # include
complete Karaf log
```
diff --git a/itests/archive-it-run.sh b/itests/archive-it-run.sh
index cb9fafa60..afb161ef9 100755
--- a/itests/archive-it-run.sh
+++ b/itests/archive-it-run.sh
@@ -48,6 +48,8 @@
# - llm-it-run-analysis-guide.md, expected-karaf-log-patterns.txt
# - exam/.../karaf-unexpected-candidates.log (errors not matching expected
patterns)
# - test-results.tsv, run-summary.properties, failed-tests.txt (LLM-friendly
per-run test manifest)
+# - it-run-operator-note.txt (auto-generated operator note from build.sh)
+# - memory-samples.tsv, memory-summary.txt (JVM/system memory observed
during IT run)
# - run-context.txt, run-config/it-run-trace.properties (build/Maven/options
trace)
# - archives/runs-index.tsv (updated each capture — cross-run comparison
index)
# - comparison-last-3.txt, archives/latest-comparison.txt (auto when 2+
captures exist)
@@ -64,6 +66,8 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "$SCRIPT_DIR/lib/it-run-bootstrap.sh"
# shellcheck source=lib/it-run-karaf.sh disable=SC1091
source "$SCRIPT_DIR/lib/it-run-karaf.sh"
+# shellcheck source=lib/it-run-memory.sh disable=SC1091
+source "$SCRIPT_DIR/lib/it-run-memory.sh"
# shellcheck source=lib/it-run-context.sh disable=SC1091
source "$SCRIPT_DIR/lib/it-run-context.sh"
@@ -85,6 +89,7 @@ RUN_CONFIG_FILES=(
elasticsearch-port.properties
opensearch-port.properties
it-run-trace.properties
+ it-run-operator-note.txt
)
ENGINE_LOG_TREES=(
@@ -113,7 +118,7 @@ usage() {
echo " (default) Unexploded directory:
archives/it-run-YYYYMMDD-HHMMSS/"
echo " --tar Write a .tar.gz instead (default name under
archives/)"
echo " -o PATH Output directory, or .tar.gz / .tgz archive path"
- echo " -m, --message Operator note about run context (quoted string)"
+ echo " -m, --message Operator note (default: it-run-operator-note.txt
from build.sh)"
echo " --message-file Read operator note from a file"
echo " --full-karaf Include complete karaf.log and rollover segments
(default: tail + filtered errors)"
echo " --no-compare Skip auto compare of last 3 captures (default: on
when 2+ runs exist)"
@@ -121,6 +126,17 @@ usage() {
exit 1
}
+
+load_default_operator_note() {
+ if [ -n "$RUN_MESSAGE" ]; then
+ return 0
+ fi
+ if RUN_MESSAGE="$(it_load_default_operator_note "$TARGET_DIR"
"$SCRIPT_DIR" 2>/dev/null)"; then
+ ui_detail "Using auto-generated operator note from
$IT_OPERATOR_NOTE_FILE"
+ return 0
+ fi
+}
+
parse_args() {
while [ $# -gt 0 ]; do
case "$1" in
@@ -373,6 +389,10 @@ write_run_summary() {
echo "tests.skipped=$(it_failsafe_summary_count "$summary_xml"
skipped)"
fi
echo "failed.tests.count=$failed_count"
+ if [ -f "$(target_path "$IT_MEMORY_SUMMARY")" ]; then
+ grep -E '^memory\.(peak|min|warning|samples|karaf|search)\.' \
+ "$(target_path "$IT_MEMORY_SUMMARY")" 2>/dev/null || true
+ fi
if [ -n "$RUN_FINGERPRINT" ]; then
echo "${IT_RUN_FINGERPRINT_FIELD}=$RUN_FINGERPRINT"
fi
@@ -416,11 +436,24 @@ archive_engine_log_trees() {
done
}
+archive_memory_artifacts() {
+ local name samples summary
+ for name in "$IT_MEMORY_SAMPLES" "$IT_MEMORY_SUMMARY"
"$IT_MEMORY_SAMPLER_LOG"; do
+ copy_file "$(target_path "$name")" "$name"
+ done
+ samples="$(target_path "$IT_MEMORY_SAMPLES")"
+ summary="$(target_path "$IT_MEMORY_SUMMARY")"
+ if [ -f "$samples" ] && [ ! -f "$summary" ]; then
+ it_memory_summarize_samples "$samples" "$summary" && copy_file
"$summary" "$IT_MEMORY_SUMMARY"
+ fi
+}
+
archive_test_artifacts() {
archive_report_trees
capture_test_manifest
archive_run_config_files
archive_engine_log_trees
+ archive_memory_artifacts
write_run_context
}
@@ -493,6 +526,7 @@ finalize_output() {
main() {
it_run_entry_init "$SCRIPT_DIR"
parse_args "$@"
+ load_default_operator_note
require_target_dir
reject_duplicate_archive
diff --git a/itests/lib/it-run-context.sh b/itests/lib/it-run-context.sh
index f09f73782..c7bbe52d5 100644
--- a/itests/lib/it-run-context.sh
+++ b/itests/lib/it-run-context.sh
@@ -20,6 +20,7 @@
# Expects archive hooks: target_path, staging_path, log_staged_file,
mark_included,
# and globals: SCRIPT_DIR, REPO_ROOT, TARGET_DIR, MANIFEST, OUTPUT,
RUN_MESSAGE,
# RUN_ID, FULL_KARAF, CREATE_TAR, RUN_CONTEXT_REL, RUN_TRACE_FILE.
+# Requires it-run-memory.sh to be sourced by the caller (archive-it-run.sh).
context_append_inferred_maven_properties() {
local engine
@@ -72,6 +73,7 @@ context_append_system_snapshot() {
if [ "$(uname -s 2>/dev/null)" = Darwin ] && command -v memory_pressure
>/dev/null 2>&1; then
echo "snapshot.memory_pressure=$(memory_pressure 2>/dev/null | head
-1)"
fi
+ it_memory_append_system_snapshot
}
context_emit_manifest_git_lines() {
@@ -151,6 +153,10 @@ write_run_context() {
echo
echo "## System snapshot (at archive time — not at IT start)"
context_append_system_snapshot
+ echo
+ it_memory_append_context_section \
+ "$(target_path "$IT_MEMORY_SUMMARY")" \
+ "$(target_path "$IT_MEMORY_SAMPLES")"
} > "$out"
log_staged_file "$RUN_CONTEXT_REL" "Wrote $RUN_CONTEXT_REL"
mark_included
diff --git a/itests/lib/it-run-memory.sh b/itests/lib/it-run-memory.sh
new file mode 100644
index 000000000..1c8f43466
--- /dev/null
+++ b/itests/lib/it-run-memory.sh
@@ -0,0 +1,651 @@
+################################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+# JVM and system memory sampling helpers for integration test runs.
+# Requires lib/it-run.sh to be sourced first (shared engine + POM helpers).
+
+IT_MEMORY_SAMPLES="memory-samples.tsv"
+IT_MEMORY_SUMMARY="memory-summary.txt"
+IT_MEMORY_SAMPLER_PID="memory-sampler.pid"
+IT_MEMORY_SAMPLER_STOP="memory-sampler.stop"
+IT_MEMORY_SAMPLER_LOG="memory-sampler.log"
+IT_MEMORY_SAMPLER_CACHE="memory-sampler.cache"
+IT_MEMORY_SWAP_PRESSURE_MB=2048
+
+IT_MEMORY_TSV_HEADER=$'timestamp_utc\tkaraf_pid\tkaraf_heap_used_mb\tkaraf_heap_max_mb\tkaraf_gct_s\tes_heap_used_mb\tes_heap_max_mb\tdocker_rss_mb\tsystem_mem_available_mb\tsystem_swap_used_mb\tsystem_load_1m'
+
+_IT_MEMORY_OS=""
+
+it_memory_os_name() {
+ if [ -z "$_IT_MEMORY_OS" ]; then
+ _IT_MEMORY_OS="$(uname -s 2>/dev/null || echo unknown)"
+ fi
+ echo "$_IT_MEMORY_OS"
+}
+
+it_memory_is_linux() {
+ [ "$(it_memory_os_name)" = Linux ]
+}
+
+it_memory_is_darwin() {
+ [ "$(it_memory_os_name)" = Darwin ]
+}
+
+it_memory_parse_heap_setting_mb() {
+ local value="${1:-}"
+ local number suffix
+
+ value="$(echo "$value" | tr -d '[:space:]')"
+ if [ -z "$value" ]; then
+ echo "0"
+ return
+ fi
+
+ suffix="${value: -1}"
+ number="${value%?}"
+ case "$suffix" in
+ g | G) awk -v n="$number" 'BEGIN { printf "%d", int(n * 1024 + 0.5) }'
;;
+ m | M) awk -v n="$number" 'BEGIN { printf "%d", int(n + 0.5) }' ;;
+ [0-9]) awk -v n="$value" 'BEGIN { printf "%d", int(n / 1048576 + 0.5)
}' ;;
+ *) echo "0" ;;
+ esac
+}
+
+it_memory_parse_jvm_flag_bytes() {
+ local output="$1"
+ local flag="$2"
+
+ echo "$output" | sed -n
"s/.*${flag}[[:space:]]*=[[:space:]]*\([0-9][0-9]*\).*/\1/p" | head -1
+}
+
+it_memory_read_trace_heap_mb() {
+ local target_dir="$1"
+ local trace_field="$2"
+ local pom_profile="$3"
+ local pom_field="$4"
+ local script_dir trace_file configured
+
+ script_dir="$(cd "$(dirname "$target_dir")" && pwd)"
+ trace_file="$target_dir/it-run-trace.properties"
+ if [ ! -f "$script_dir/pom.xml" ]; then
+ echo "0"
+ return
+ fi
+ configured="$(it_resolve_configured_heap "$trace_file" "$trace_field"
"$script_dir/pom.xml" "$pom_profile" "$pom_field")"
+ it_memory_parse_heap_setting_mb "$configured"
+}
+
+it_memory_detect_karaf_max_mb() {
+ local target_dir="$1"
+ local pid="$2"
+ local cap_mb_from_jstat="${3:-0}"
+ local max_bytes max_mb jinfo_out jcmd_out profile
+
+ max_mb="0"
+
+ if command -v jinfo >/dev/null 2>&1; then
+ jinfo_out="$(jinfo -flag MaxHeapSize "$pid" 2>/dev/null || true)"
+ max_bytes="$(it_memory_parse_jvm_flag_bytes "$jinfo_out"
"MaxHeapSize")"
+ if [ -n "$max_bytes" ]; then
+ max_mb="$(it_memory_mb_from_bytes "$max_bytes")"
+ fi
+ fi
+
+ if [ "$max_mb" = "0" ] && command -v jcmd >/dev/null 2>&1; then
+ jcmd_out="$(jcmd "$pid" VM.flags 2>/dev/null || true)"
+ max_bytes="$(it_memory_parse_jvm_flag_bytes "$jcmd_out" "MaxHeapSize")"
+ if [ -n "$max_bytes" ]; then
+ max_mb="$(it_memory_mb_from_bytes "$max_bytes")"
+ fi
+ fi
+
+ if [ "$max_mb" = "0" ] && [ "$cap_mb_from_jstat" -gt 0 ] 2>/dev/null; then
+ max_mb="$cap_mb_from_jstat"
+ fi
+
+ if [ "$max_mb" = "0" ]; then
+ profile="$(it_search_engine_pom_profile
"$(it_memory_resolve_search_engine "$target_dir")")"
+ max_mb="$(it_memory_read_trace_heap_mb "$target_dir" "karaf.heap"
"$profile" "karaf.heap")"
+ fi
+
+ echo "$max_mb"
+}
+
+it_memory_jstat_gc_metrics() {
+ local pid="$1"
+
+ if ! command -v jstat >/dev/null 2>&1; then
+ echo -e "0\t0\t0"
+ return
+ fi
+
+ jstat -gc "$pid" 2>/dev/null | awk '
+ NR == 1 {
+ for (i = 1; i <= NF; i++) {
+ h[$i] = i
+ }
+ next
+ }
+ NR == 2 {
+ if ("S0U" in h) {
+ used_kb = $(h["S0U"]) + $(h["S1U"]) + $(h["EU"]) + $(h["OU"])
+ } else {
+ used_kb = $3 + $4 + $6 + $8
+ }
+ if ("S0C" in h) {
+ cap_kb = $(h["S0C"]) + $(h["S1C"]) + $(h["EC"]) + $(h["OC"])
+ } else {
+ cap_kb = $1 + $2 + $5 + $7
+ }
+ if ("GCT" in h) {
+ gct = $(h["GCT"]) + 0
+ } else {
+ gct = $NF + 0
+ }
+ printf "%d\t%d\t%.3f", int(used_kb * 1024 / 1048576 + 0.5),
int(cap_kb * 1024 / 1048576 + 0.5), gct
+ }'
+}
+
+it_memory_system_mem_available_mb() {
+ local pagesize
+
+ if it_memory_is_linux && command -v free >/dev/null 2>&1; then
+ free -m 2>/dev/null | awk '/^Mem:/ {
+ if ($7 != "") {
+ print $7
+ } else {
+ print $4
+ }
+ exit
+ }'
+ return
+ fi
+
+ if it_memory_is_darwin && command -v vm_stat >/dev/null 2>&1; then
+ pagesize="$(sysctl -n hw.pagesize 2>/dev/null || echo 4096)"
+ vm_stat 2>/dev/null | awk -v ps="$pagesize" '
+ /Pages free/ { gsub(/\./, "", $3); free = $3 + 0 }
+ /Pages inactive/ { gsub(/\./, "", $3); inactive = $3 + 0 }
+ END {
+ printf "%d", int((free + inactive) * ps / 1048576 + 0.5)
+ }'
+ return
+ fi
+
+ echo "0"
+}
+
+it_memory_system_swap_used_mb() {
+ if it_memory_is_linux && command -v free >/dev/null 2>&1; then
+ free -m 2>/dev/null | awk '/^Swap:/ { print $3; exit }'
+ return
+ fi
+
+ if it_memory_is_darwin; then
+ sysctl -n vm.swapusage 2>/dev/null | sed -n 's/.* used =
\([0-9.]*\)M.*/\1/p' | awk '{ printf "%d", int($1 + 0.5) }'
+ return
+ fi
+
+ echo "0"
+}
+
+it_memory_system_load_1m() {
+ if it_memory_is_linux && [ -r /proc/loadavg ]; then
+ awk '{ print $1 + 0 }' /proc/loadavg
+ return
+ fi
+
+ if it_memory_is_darwin && command -v sysctl >/dev/null 2>&1; then
+ sysctl -n vm.loadavg 2>/dev/null | tr -d '{}\n' | awk '{ print $1 + 0
}'
+ return
+ fi
+
+ if command -v uptime >/dev/null 2>&1; then
+ uptime 2>/dev/null | sed -n 's/.*load average[s]*: //p' | cut -d, -f1
| tr -d ' '
+ return
+ fi
+
+ echo "0"
+}
+
+it_memory_sampler_cache_file() {
+ echo "$1/$IT_MEMORY_SAMPLER_CACHE"
+}
+
+it_memory_read_cache_field() {
+ local cache_file="$1"
+ local field="$2"
+ if [ ! -f "$cache_file" ]; then
+ return 1
+ fi
+ grep -m1 "^${field}=" "$cache_file" 2>/dev/null | cut -d= -f2-
+}
+
+it_memory_write_cache_field() {
+ local cache_file="$1"
+ local field="$2"
+ local value="$3"
+ local tmp
+
+ mkdir -p "$(dirname "$cache_file")"
+ touch "$cache_file"
+ tmp="$(mktemp "${TMPDIR:-/tmp}/unomi-mem-cache.XXXXXX")"
+ grep -v "^${field}=" "$cache_file" > "$tmp" 2>/dev/null || true
+ printf '%s=%s\n' "$field" "$value" >> "$tmp"
+ mv "$tmp" "$cache_file"
+}
+
+it_memory_clear_sampler_cache() {
+ rm -f "$(it_memory_sampler_cache_file "$1")"
+}
+
+it_memory_resolve_search_engine() {
+ local target_dir="$1"
+ local engine
+
+ if [ -n "${IT_SEARCH_ENGINE:-}" ]; then
+ echo "$IT_SEARCH_ENGINE"
+ return
+ fi
+ engine="$(it_infer_search_engine "$target_dir")"
+ if [ -n "$engine" ] && [ "$engine" != "unknown" ]; then
+ echo "$engine"
+ return
+ fi
+ echo "elasticsearch"
+}
+
+it_memory_resolve_docker_container() {
+ local target_dir="$1"
+ local cache_file engine container default_container
+
+ cache_file="$(it_memory_sampler_cache_file "$target_dir")"
+ container="$(it_memory_read_cache_field "$cache_file" docker_container ||
true)"
+ if [ -n "$container" ]; then
+ echo "$container"
+ return
+ fi
+
+ engine="$(it_memory_resolve_search_engine "$target_dir")"
+ default_container="$(it_search_engine_docker_container "$engine")"
+
+ if command -v docker >/dev/null 2>&1; then
+ container="$(docker ps --filter "name=${default_container}" --format
'{{.Names}}' 2>/dev/null | head -1)"
+ fi
+
+ if [ -z "$container" ]; then
+ echo "$default_container"
+ return
+ fi
+
+ it_memory_write_cache_field "$cache_file" docker_container "$container"
+ echo "$container"
+}
+
+it_memory_mb_from_bytes() {
+ local bytes="$1"
+ if [ -z "$bytes" ] || [ "$bytes" = "0" ]; then
+ echo "0"
+ return
+ fi
+ echo $(( (bytes + 524288) / 1048576 ))
+}
+
+it_memory_json_number() {
+ local json="$1"
+ local field="$2"
+ echo "$json" | sed -n "s/.*\"${field}\":\([0-9][0-9]*\).*/\1/p" | head -1
+}
+
+it_memory_resolve_search_port() {
+ local target_dir="$1"
+ local engine port_file port_line
+
+ engine="$(it_memory_resolve_search_engine "$target_dir")"
+ port_file="$target_dir/$(it_search_engine_port_properties_file "$engine")"
+
+ if [ -f "$port_file" ]; then
+ port_line="$(grep -m1 '\.port=' "$port_file" 2>/dev/null | cut -d=
-f2- | tr -d '[:space:]')"
+ if [ -n "$port_line" ]; then
+ echo "$port_line"
+ return
+ fi
+ fi
+
+ if [ -n "${IT_SEARCH_PORT:-}" ]; then
+ echo "$IT_SEARCH_PORT"
+ return
+ fi
+
+ it_search_engine_default_port "$engine"
+}
+
+it_memory_parse_docker_mem_to_mb() {
+ local raw="${1:-}"
+
+ case "$raw" in
+ *GiB) echo "${raw%GiB}" | awk '{printf "%d", $1 * 1024 + 0.5}' ;;
+ *MiB) echo "${raw%MiB}" | awk '{printf "%d", $1 + 0.5}' ;;
+ *KiB) echo "${raw%KiB}" | awk '{printf "%d", int($1 / 1024 + 0.5)}' ;;
+ *G) echo "${raw%G}" | awk '{printf "%d", $1 * 1024 + 0.5}' ;;
+ *M) echo "${raw%M}" | awk '{printf "%d", $1 + 0.5}' ;;
+ *K) echo "${raw%K}" | awk '{printf "%d", int($1 / 1024 + 0.5)}' ;;
+ *) echo "0" ;;
+ esac
+}
+
+it_memory_find_karaf_pid() {
+ pgrep -f 'org.apache.karaf.main.Main' 2>/dev/null | head -1
+}
+
+it_memory_karaf_max_mb_cached() {
+ local target_dir="$1"
+ local pid="$2"
+ local cap_mb_from_jstat="${3:-0}"
+ local cache_file cached_pid max_mb
+
+ cache_file="$(it_memory_sampler_cache_file "$target_dir")"
+ cached_pid="$(it_memory_read_cache_field "$cache_file" karaf_pid || true)"
+ if [ "$cached_pid" = "$pid" ]; then
+ max_mb="$(it_memory_read_cache_field "$cache_file" karaf_max_mb ||
true)"
+ if [ -n "$max_mb" ] && [ "$max_mb" -gt 0 ] 2>/dev/null; then
+ echo "$max_mb"
+ return
+ fi
+ fi
+
+ max_mb="$(it_memory_detect_karaf_max_mb "$target_dir" "$pid"
"$cap_mb_from_jstat")"
+ if [ -z "$max_mb" ]; then
+ max_mb="0"
+ fi
+
+ it_memory_write_cache_field "$cache_file" karaf_pid "$pid"
+ if [ "$max_mb" -gt 0 ] 2>/dev/null; then
+ it_memory_write_cache_field "$cache_file" karaf_max_mb "$max_mb"
+ else
+ it_memory_write_cache_field "$cache_file" karaf_max_mb ""
+ fi
+ echo "$max_mb"
+}
+
+it_memory_karaf_stats() {
+ local target_dir="$1"
+ local pid="$2"
+ local used_mb max_mb cap_mb gct_s jstat_line
+
+ if [ -z "$pid" ]; then
+ echo -e "0\t0\t0"
+ return
+ fi
+
+ if ! kill -0 "$pid" 2>/dev/null; then
+ echo -e "0\t0\t0"
+ return
+ fi
+
+ used_mb="0"
+ cap_mb="0"
+ gct_s="0"
+ jstat_line="$(it_memory_jstat_gc_metrics "$pid")"
+ if [ -n "$jstat_line" ]; then
+ read -r used_mb cap_mb gct_s <<EOF
+$jstat_line
+EOF
+ fi
+
+ max_mb="$(it_memory_karaf_max_mb_cached "$target_dir" "$pid" "$cap_mb")"
+
+ if [ -z "${used_mb:-}" ] || [ "$used_mb" = "0" ]; then
+ if command -v ps >/dev/null 2>&1; then
+ used_mb="$(ps -o rss= -p "$pid" 2>/dev/null | awk '{print
int($1/1024+0.5)}')"
+ fi
+ fi
+
+ if [ -z "${max_mb:-}" ] || [ "$max_mb" = "0" ]; then
+ max_mb="0"
+ fi
+ if [ -z "${used_mb:-}" ]; then
+ used_mb="0"
+ fi
+ if [ -z "${gct_s:-}" ]; then
+ gct_s="0"
+ fi
+
+ echo -e "${used_mb}\t${max_mb}\t${gct_s}"
+}
+
+it_memory_search_engine_stats() {
+ local port="$1"
+ local json used_bytes max_bytes
+
+ json="$(curl -sf --max-time 5 \
+
"http://localhost:${port}/_nodes/stats/jvm?filter_path=nodes.*.jvm.mem.heap_used_in_bytes,nodes.*.jvm.mem.heap_max_in_bytes";
\
+ 2>/dev/null || true)"
+ if [ -z "$json" ]; then
+ echo -e "0\t0"
+ return
+ fi
+
+ used_bytes="$(it_memory_json_number "$json" "heap_used_in_bytes")"
+ max_bytes="$(it_memory_json_number "$json" "heap_max_in_bytes")"
+ echo -e "$(it_memory_mb_from_bytes
"${used_bytes:-0}")\t$(it_memory_mb_from_bytes "${max_bytes:-0}")"
+}
+
+it_memory_docker_rss_mb() {
+ local target_dir="$1"
+ local container rss
+
+ if ! command -v docker >/dev/null 2>&1; then
+ echo "0"
+ return
+ fi
+
+ container="$(it_memory_resolve_docker_container "$target_dir")"
+ rss="$(docker stats --no-stream --format '{{.MemUsage}}' "$container"
2>/dev/null | head -1 | cut -d/ -f1 | tr -d ' ')"
+
+ it_memory_parse_docker_mem_to_mb "$rss"
+}
+
+it_memory_system_stats() {
+ local mem_available swap_used load_1m
+
+ mem_available="$(it_memory_system_mem_available_mb)"
+ swap_used="$(it_memory_system_swap_used_mb)"
+ load_1m="$(it_memory_system_load_1m)"
+
+ echo -e "${mem_available:-0}\t${swap_used:-0}\t${load_1m:-0}"
+}
+
+it_memory_sample_once() {
+ local target_dir="$1"
+ local port="${2:-}"
+ local karaf_pid karaf_line es_line sys_line
+
+ port="${port:-$(it_memory_resolve_search_port "$target_dir")}"
+ karaf_pid="$(it_memory_find_karaf_pid)"
+ karaf_line="$(it_memory_karaf_stats "$target_dir" "$karaf_pid")"
+ es_line="$(it_memory_search_engine_stats "$port")"
+ sys_line="$(it_memory_system_stats)"
+
+ printf '%s\t%s\t%s\t%s\t%s\t%s\n' \
+ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+ "${karaf_pid:-0}" \
+ "$karaf_line" \
+ "$es_line" \
+ "$(it_memory_docker_rss_mb "$target_dir")" \
+ "$sys_line"
+}
+
+it_memory_write_samples_header() {
+ local samples_file="$1"
+ if [ ! -f "$samples_file" ] || [ ! -s "$samples_file" ]; then
+ printf '%s\n' "$IT_MEMORY_TSV_HEADER" > "$samples_file"
+ fi
+}
+
+it_memory_summarize_samples() {
+ local samples_file="$1"
+ local summary_file="$2"
+
+ if [ ! -f "$samples_file" ] || [ ! -s "$samples_file" ]; then
+ return 1
+ fi
+
+ awk -F'\t' -v summary="$summary_file" -v
swap_pressure_mb="$IT_MEMORY_SWAP_PRESSURE_MB" '
+ NR == 1 { next }
+ NF < 11 { next }
+ {
+ samples++
+ if ($2+0 > 0) {
+ if ($3+0 > peak_karaf_used) peak_karaf_used = $3+0
+ if ($4+0 > peak_karaf_max) peak_karaf_max = $4+0
+ if ($5+0 > peak_karaf_gct_s) peak_karaf_gct_s = $5+0
+ }
+ if ($6+0 > peak_es_used) peak_es_used = $6+0
+ if ($7+0 > peak_es_max) peak_es_max = $7+0
+ if ($8+0 > peak_docker_rss) peak_docker_rss = $8+0
+ if ($9+0 > 0 && (min_mem_avail == 0 || $9+0 < min_mem_avail))
min_mem_avail = $9+0
+ if ($10+0 > peak_swap) peak_swap = $10+0
+ if ($11+0 > peak_load) peak_load = $11+0
+ if (samples == 1) first_swap = $10+0
+ last_swap = $10+0
+ }
+ END {
+ if (samples == 0) exit 1
+ karaf_headroom = (peak_karaf_max > 0 ? (peak_karaf_max -
peak_karaf_used) * 100 / peak_karaf_max : 0)
+ es_headroom = (peak_es_max > 0 ? (peak_es_max - peak_es_used) *
100 / peak_es_max : 0)
+ swap_pressure = (peak_swap+0 > 0 && min_mem_avail+0 > 0 &&
min_mem_avail+0 < swap_pressure_mb+0) || (last_swap+0 > first_swap+0 + 256)
+ printf("# Integration test memory summary (generated from %s)\n",
FILENAME) > summary
+ printf("memory.samples.count=%d\n", samples) >> summary
+ printf("memory.peak.karaf.heap.used.mb=%d\n", peak_karaf_used+0)
>> summary
+ printf("memory.peak.karaf.heap.max.mb=%d\n", peak_karaf_max+0) >>
summary
+ printf("memory.peak.karaf.gct.s=%.2f\n", peak_karaf_gct_s+0) >>
summary
+ printf("memory.peak.search.heap.used.mb=%d\n", peak_es_used+0) >>
summary
+ printf("memory.peak.search.heap.max.mb=%d\n", peak_es_max+0) >>
summary
+ printf("memory.peak.docker.rss.mb=%d\n", peak_docker_rss+0) >>
summary
+ printf("memory.min.system.mem.available.mb=%d\n", min_mem_avail+0)
>> summary
+ printf("memory.peak.system.swap.used.mb=%d\n", peak_swap+0) >>
summary
+ printf("memory.peak.system.load.1m=%.2f\n", peak_load+0) >> summary
+ printf("memory.karaf.headroom.pct=%d\n", karaf_headroom+0) >>
summary
+ printf("memory.search.headroom.pct=%d\n", es_headroom+0) >> summary
+ if (swap_pressure) {
+ printf("memory.warning.swap.detected=true\n") >> summary
+ } else {
+ printf("memory.warning.swap.detected=false\n") >> summary
+ }
+ if (peak_karaf_max > 0 && peak_karaf_used * 100 / peak_karaf_max
>= 85) {
+ printf("memory.warning.karaf.heap.high=true\n") >> summary
+ } else {
+ printf("memory.warning.karaf.heap.high=false\n") >> summary
+ }
+ if (peak_es_max > 0 && peak_es_used * 100 / peak_es_max >= 85) {
+ printf("memory.warning.search.heap.high=true\n") >> summary
+ } else {
+ printf("memory.warning.search.heap.high=false\n") >> summary
+ }
+ }
+ ' "$samples_file"
+}
+
+it_memory_append_context_section() {
+ local summary_file="$1"
+ local samples_file="$2"
+
+ if [ -f "$summary_file" ]; then
+ echo "## Memory summary (observed during IT run)"
+ cat "$summary_file"
+ echo
+ elif [ -f "$samples_file" ]; then
+ echo "## Memory samples"
+ echo "# memory-summary.txt missing; raw samples available at
$(basename "$samples_file")"
+ echo "memory.samples.lines=$(($(wc -l < "$samples_file" 2>/dev/null |
tr -d ' ') - 1))"
+ echo
+ fi
+}
+
+it_memory_append_system_snapshot() {
+ if it_memory_is_linux && command -v free >/dev/null 2>&1; then
+ echo "snapshot.mem.linux=$(free -h 2>/dev/null | awk '/^Mem:/ {print
$0}')"
+ echo "snapshot.swap.linux=$(free -h 2>/dev/null | awk '/^Swap:/ {print
$0}')"
+ fi
+ echo "snapshot.mem.available.mb=$(it_memory_system_mem_available_mb)"
+ echo "snapshot.swap.used.mb=$(it_memory_system_swap_used_mb)"
+ echo "snapshot.load.1m=$(it_memory_system_load_1m)"
+}
+
+it_memory_verify() {
+ local target_dir="$1"
+ local script_dir="${2:-$(cd "$(dirname "$target_dir")" && pwd)}"
+ local pom="$script_dir/pom.xml"
+ local failures=0 karaf_pid sample_line
+
+ _it_memory_verify_eq() {
+ if [ "$2" = "$3" ]; then
+ echo "OK $1"
+ else
+ echo "FAIL $1 (expected '$2', got '$3')"
+ failures=$((failures + 1))
+ fi
+ }
+
+ _it_memory_verify_gt_zero() {
+ if awk -v v="${2:-0}" 'BEGIN { exit !(v + 0 > 0) }'; then
+ echo "OK $1 ($2)"
+ else
+ echo "FAIL $1 (expected > 0, got '${2:-0}')"
+ failures=$((failures + 1))
+ fi
+ }
+
+ echo "Memory sampling verification on $(it_memory_os_name)"
+ _it_memory_verify_eq "parse 2g" "2048" "$(it_memory_parse_heap_setting_mb
2g)"
+ _it_memory_verify_eq "parse 1536m" "1536"
"$(it_memory_parse_heap_setting_mb 1536m)"
+ _it_memory_verify_eq "parse docker GiB" "2048"
"$(it_memory_parse_docker_mem_to_mb 2GiB)"
+ _it_memory_verify_eq "parse docker MiB" "512"
"$(it_memory_parse_docker_mem_to_mb 512MiB)"
+ _it_memory_verify_eq "parse docker KiB" "1"
"$(it_memory_parse_docker_mem_to_mb 1024KiB)"
+ _it_memory_verify_eq "parse jinfo flag" "2147483648" \
+ "$(it_memory_parse_jvm_flag_bytes '-XX:MaxHeapSize=2147483648'
'MaxHeapSize')"
+ _it_memory_verify_eq "search engine default port" "9400"
"$(it_search_engine_default_port elasticsearch)"
+ _it_memory_verify_eq "search engine container" "itests-elasticsearch"
"$(it_search_engine_docker_container elasticsearch)"
+ _it_memory_verify_eq "pom elasticsearch.heap" "4g" \
+ "$(it_read_pom_profile_property "$pom" elasticsearch
elasticsearch.heap 2>/dev/null || true)"
+ _it_memory_verify_eq "pom karaf.heap" "2g" \
+ "$(it_read_pom_profile_property "$pom" elasticsearch karaf.heap
2>/dev/null || true)"
+ _it_memory_verify_gt_zero "system load 1m" "$(it_memory_system_load_1m)"
+ _it_memory_verify_gt_zero "system mem available mb"
"$(it_memory_system_mem_available_mb)"
+ echo "OK system swap used mb ($(it_memory_system_swap_used_mb))"
+
+ karaf_pid="$(it_memory_find_karaf_pid || true)"
+ if [ -n "$karaf_pid" ]; then
+ echo "Karaf PID $karaf_pid — live checks"
+ sample_line="$(it_memory_sample_once "$target_dir")"
+ _it_memory_verify_gt_zero "sample karaf_heap_max_mb" "$(echo
"$sample_line" | awk -F'\t' '{print $4}')"
+ _it_memory_verify_gt_zero "sample system_load_1m parseable" \
+ "$(echo "$sample_line" | awk -F'\t' '{print ($11+0 > 0)}')"
+ echo "Sample: $sample_line"
+ else
+ echo "SKIP live Karaf checks (no Karaf process)"
+ fi
+
+ if [ "$failures" -eq 0 ]; then
+ echo "All checks passed."
+ return 0
+ fi
+ echo "$failures check(s) failed."
+ return 1
+}
diff --git a/itests/lib/it-run.sh b/itests/lib/it-run.sh
index ed50a8136..dbe9b83c4 100644
--- a/itests/lib/it-run.sh
+++ b/itests/lib/it-run.sh
@@ -37,7 +37,9 @@ IT_ENGINE_PORT_FILES=(
elasticsearch-port.properties
opensearch-port.properties
)
-IT_RUN_CONTEXT_NO_OPERATOR_NOTE='(none — pass --message "..." to describe run
conditions, e.g. heavy swap, CI runner, single-test rerun)'
+IT_RUN_CONTEXT_NO_OPERATOR_NOTE='(none — auto-generated note missing; run via
./build.sh --integration-tests or pass -m "...")'
+IT_RUN_TRACE="it-run-trace.properties"
+IT_OPERATOR_NOTE_FILE="it-run-operator-note.txt"
it_run_lib_init() {
IT_SCRIPT_DIR="$1"
@@ -77,16 +79,35 @@ it_read_pom_profile_property() {
awk -v profile="$profile" -v prop="$property" '
$0 ~ "<id>" profile "</id>" { in_profile=1 }
- in_profile && $0 ~ "<" prop ">" {
- gsub(/.*<[^>]+>/, "")
- gsub(/<.*/, "")
- print
+ in_profile && index($0, "<" prop ">") {
+ start = index($0, "<" prop ">") + length(prop) + 2
+ rest = substr($0, start)
+ end = index(rest, "<")
+ if (end > 0) {
+ print substr(rest, 1, end - 1)
+ }
exit
}
in_profile && $0 ~ "</profile>" { exit }
' "$pom"
}
+it_resolve_configured_heap() {
+ local trace_file="$1"
+ local trace_field="$2"
+ local pom="$3"
+ local profile="$4"
+ local pom_field="$5"
+ local configured
+
+ configured="$(it_read_properties_field "$trace_file" "$trace_field")"
+ if [ -n "$configured" ]; then
+ echo "$configured"
+ return
+ fi
+ it_read_pom_profile_property "$pom" "$profile" "$pom_field"
+}
+
it_infer_search_engine() {
local target_dir="$1"
@@ -103,6 +124,41 @@ it_infer_search_engine() {
fi
}
+it_search_engine_pom_profile() {
+ case "$1" in
+ opensearch) echo "opensearch" ;;
+ *) echo "elasticsearch" ;;
+ esac
+}
+
+it_search_engine_heap_pom_field() {
+ case "$1" in
+ opensearch) echo "opensearch.heap" ;;
+ *) echo "elasticsearch.heap" ;;
+ esac
+}
+
+it_search_engine_docker_container() {
+ case "$1" in
+ opensearch) echo "itests-opensearch" ;;
+ *) echo "itests-elasticsearch" ;;
+ esac
+}
+
+it_search_engine_port_properties_file() {
+ case "$1" in
+ opensearch) echo "opensearch-port.properties" ;;
+ *) echo "elasticsearch-port.properties" ;;
+ esac
+}
+
+it_search_engine_default_port() {
+ case "$1" in
+ opensearch) echo "9401" ;;
+ *) echo "9400" ;;
+ esac
+}
+
it_run_label() {
basename "$1"
}
@@ -469,3 +525,137 @@ it_update_runs_index() {
} > "$index"
ui_detail "Updated $index"
}
+
+it_format_configured_heap() {
+ local trace_file="$1"
+ local trace_field="$2"
+ local pom="$3"
+ local profile="$4"
+ local pom_field="$5"
+ local trace_value resolved
+
+ trace_value="$(it_read_properties_field "$trace_file" "$trace_field")"
+ if [ -n "$trace_value" ]; then
+ echo "$trace_value"
+ return
+ fi
+ resolved="$(it_resolve_configured_heap "$trace_file" "$trace_field" "$pom"
"$profile" "$pom_field")"
+ if [ -n "$resolved" ]; then
+ echo "${resolved} (pom default)"
+ else
+ echo "unknown"
+ fi
+}
+
+it_generate_operator_note() {
+ local target_dir="$1"
+ local script_dir="$2"
+ local trace_file="$target_dir/$IT_RUN_TRACE"
+ local memory_summary="$target_dir/${IT_MEMORY_SUMMARY:-memory-summary.txt}"
+ local failsafe_summary="$target_dir/failsafe-reports/failsafe-summary.xml"
+ local engine profile search_heap karaf_heap exit_code single_test
+ local failures errors skipped completed started completed_ts host ci
+ local swap_warn search_high karaf_high
+
+ if [ ! -f "$trace_file" ]; then
+ return 1
+ fi
+
+ engine="$(it_read_properties_field "$trace_file" search.engine)"
+ if [ -z "$engine" ]; then
+ engine="$(it_infer_search_engine "$target_dir")"
+ fi
+ profile="$(it_search_engine_pom_profile "$engine")"
+ search_heap="$(it_format_configured_heap "$trace_file" search.heap
"$script_dir/pom.xml" "$profile" "$(it_search_engine_heap_pom_field
"$engine")")"
+ karaf_heap="$(it_format_configured_heap "$trace_file" karaf.heap
"$script_dir/pom.xml" "$profile" karaf.heap)"
+ exit_code="$(it_read_properties_field "$trace_file" maven.exit.code)"
+ single_test="$(it_read_properties_field "$trace_file" single.test)"
+ started="$(it_read_properties_field "$trace_file" trace.started)"
+ completed_ts="$(it_read_properties_field "$trace_file" trace.completed)"
+ host="$(it_read_properties_field "$trace_file" host)"
+ if [ -z "$host" ]; then
+ host="$(it_hostname)"
+ fi
+ if [ -n "${GITHUB_ACTIONS:-}" ]; then
+ ci="yes (${GITHUB_WORKFLOW:-workflow}/${GITHUB_JOB:-job})"
+ else
+ ci="no"
+ fi
+
+ failures="$(it_failsafe_summary_count "$failsafe_summary" failures)"
+ errors="$(it_failsafe_summary_count "$failsafe_summary" errors)"
+ skipped="$(it_failsafe_summary_count "$failsafe_summary" skipped)"
+ completed="$(it_failsafe_summary_count "$failsafe_summary" completed)"
+
+ swap_warn="$(it_read_properties_field "$memory_summary"
memory.warning.swap.detected)"
+ search_high="$(it_read_properties_field "$memory_summary"
memory.warning.search.heap.high)"
+ karaf_high="$(it_read_properties_field "$memory_summary"
memory.warning.karaf.heap.high)"
+
+ echo "Auto-generated operator note ($(it_utc_now))"
+ if [ "$exit_code" = "0" ]; then
+ echo "Outcome: PASS (maven exit 0)"
+ elif [ -n "$exit_code" ]; then
+ echo "Outcome: FAIL (maven exit ${exit_code})"
+ else
+ echo "Outcome: unknown (run still in progress or trace incomplete)"
+ fi
+ echo "Search engine: ${engine}"
+ echo "Heaps: search=${search_heap}, karaf=${karaf_heap}"
+ echo "Host: ${host}; CI: ${ci}"
+ if [ -n "$started" ] && [ -n "$completed_ts" ]; then
+ echo "Run window: ${started} → ${completed_ts}"
+ elif [ -n "$started" ]; then
+ echo "Run started: ${started}"
+ fi
+ if [ -n "$single_test" ]; then
+ echo "Scope: single test ${single_test}"
+ else
+ echo "Scope: full AllITs suite"
+ fi
+ if [ -n "$completed" ]; then
+ echo "Tests: completed=${completed:-0} failures=${failures:-0}
errors=${errors:-0} skipped=${skipped:-0}"
+ fi
+ if [ -f "$memory_summary" ]; then
+ echo "Memory peaks: karaf=$(it_read_properties_field "$memory_summary"
memory.peak.karaf.heap.used.mb)/$(it_read_properties_field "$memory_summary"
memory.peak.karaf.heap.max.mb) MB, karaf GCT=$(it_read_properties_field
"$memory_summary" memory.peak.karaf.gct.s)s, search=$(it_read_properties_field
"$memory_summary" memory.peak.search.heap.used.mb)/$(it_read_properties_field
"$memory_summary" memory.peak.search.heap.max.mb) MB,
swap=$(it_read_properties_field "$memory_summary" mem [...]
+ if [ "$swap_warn" = "true" ]; then
+ echo "Memory alert: swap detected during run — consider lowering
JVM heaps or using a larger runner"
+ fi
+ if [ "$search_high" = "true" ]; then
+ echo "Memory alert: search engine heap usage high (>=85% of max) —
may be GC-bound"
+ fi
+ if [ "$karaf_high" = "true" ]; then
+ echo "Memory alert: Karaf heap usage high (>=85% of max) — may be
GC-bound"
+ fi
+ else
+ echo "Memory peaks: (no memory-summary.txt — sampler disabled or run
archived before finalize)"
+ fi
+}
+
+it_write_operator_note_file() {
+ local target_dir="$1"
+ local script_dir="$2"
+ local out="$target_dir/$IT_OPERATOR_NOTE_FILE"
+
+ mkdir -p "$target_dir"
+ if ! it_generate_operator_note "$target_dir" "$script_dir" > "$out"; then
+ rm -f "$out"
+ return 1
+ fi
+}
+
+it_load_default_operator_note() {
+ local target_dir="$1"
+ local script_dir="$2"
+ local note_file="$target_dir/$IT_OPERATOR_NOTE_FILE"
+
+ if [ -f "$note_file" ]; then
+ cat "$note_file"
+ return 0
+ fi
+ if [ -f "$target_dir/$IT_RUN_TRACE" ]; then
+ it_write_operator_note_file "$target_dir" "$script_dir" || return 1
+ cat "$note_file"
+ return 0
+ fi
+ return 1
+}
diff --git a/itests/sample-it-memory.sh b/itests/sample-it-memory.sh
new file mode 100755
index 000000000..19e0d5d5c
--- /dev/null
+++ b/itests/sample-it-memory.sh
@@ -0,0 +1,229 @@
+#!/bin/bash
+################################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+#
+# Sample JVM and system memory while integration tests run.
+#
+# Usage:
+# ./itests/sample-it-memory.sh start [--target-dir itests/target]
[--interval 30]
+# ./itests/sample-it-memory.sh stop [--target-dir itests/target]
+# ./itests/sample-it-memory.sh sample-once [--target-dir itests/target]
+# ./itests/sample-it-memory.sh summarize [--target-dir itests/target]
+# ./itests/sample-it-memory.sh operator-note [--target-dir itests/target]
[--print-only]
+# ./itests/sample-it-memory.sh verify [--target-dir itests/target]
+# Started automatically by ./build.sh --integration-tests (disable with
--no-memory-sampler).
+#
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=lib/it-run.sh disable=SC1091
+source "$SCRIPT_DIR/lib/it-run.sh"
+# shellcheck source=lib/it-run-memory.sh disable=SC1091
+source "$SCRIPT_DIR/lib/it-run-memory.sh"
+
+TARGET_DIR="$SCRIPT_DIR/target"
+INTERVAL=30
+SEARCH_PORT=""
+PRINT_ONLY=false
+COMMAND=""
+
+usage() {
+ cat <<EOF
+Usage: $0 <command> [options]
+
+Commands:
+ start Run the sampler in the background until "stop" is called
+ stop Stop the background sampler and write memory-summary.txt
+ sample-once Print one TSV sample line to stdout
+ summarize Build memory-summary.txt from memory-samples.tsv
+ operator-note Write it-run-operator-note.txt from trace + test + memory data
+ verify Cross-platform sanity checks (macOS + Linux)
+
+Options:
+ --target-dir DIR IT target directory (default: itests/target)
+ --interval SEC Sample interval in seconds for start (default: 30)
+ --port PORT Search engine HTTP port override
+ --print-only With operator-note: print to stdout instead of writing
file
+ -h, --help Show this help
+EOF
+ exit 1
+}
+
+parse_args() {
+ COMMAND="${1:-}"
+ shift || true
+
+ while [ $# -gt 0 ]; do
+ case "$1" in
+ --target-dir)
+ shift
+ TARGET_DIR="${1:-}"
+ ;;
+ --interval)
+ shift
+ INTERVAL="${1:-30}"
+ ;;
+ --port)
+ shift
+ SEARCH_PORT="${1:-}"
+ ;;
+ --print-only)
+ PRINT_ONLY=true
+ ;;
+ -h | --help)
+ usage
+ ;;
+ *)
+ echo "Unknown option: $1" >&2
+ usage
+ ;;
+ esac
+ shift
+ done
+
+ if [ -z "$COMMAND" ]; then
+ usage
+ fi
+
+ mkdir -p "$TARGET_DIR"
+ if [ -n "$SEARCH_PORT" ]; then
+ IT_SEARCH_PORT="$SEARCH_PORT"
+ export IT_SEARCH_PORT
+ fi
+}
+
+sampler_paths() {
+ SAMPLES_FILE="$TARGET_DIR/$IT_MEMORY_SAMPLES"
+ SUMMARY_FILE="$TARGET_DIR/$IT_MEMORY_SUMMARY"
+ PID_FILE="$TARGET_DIR/$IT_MEMORY_SAMPLER_PID"
+ STOP_FILE="$TARGET_DIR/$IT_MEMORY_SAMPLER_STOP"
+ LOG_FILE="$TARGET_DIR/$IT_MEMORY_SAMPLER_LOG"
+}
+
+sampler_loop() {
+ local interval="$1"
+ sampler_paths
+ it_memory_write_samples_header "$SAMPLES_FILE"
+
+ while [ ! -f "$STOP_FILE" ]; do
+ it_memory_sample_once "$TARGET_DIR" >> "$SAMPLES_FILE" || true
+ sleep "$interval"
+ done
+}
+
+cmd_start() {
+ sampler_paths
+
+ if [ -f "$PID_FILE" ]; then
+ existing_pid="$(cat "$PID_FILE" 2>/dev/null || true)"
+ if [ -n "$existing_pid" ] && kill -0 "$existing_pid" 2>/dev/null; then
+ echo "Memory sampler already running (pid $existing_pid)" >&2
+ exit 0
+ fi
+ rm -f "$PID_FILE"
+ fi
+
+ rm -f "$STOP_FILE"
+ it_memory_clear_sampler_cache "$TARGET_DIR"
+ it_memory_write_samples_header "$SAMPLES_FILE"
+ echo "# started $(date -u +%Y-%m-%dT%H:%M:%SZ) interval=${INTERVAL}s
target=$TARGET_DIR" >> "$LOG_FILE"
+
+ (
+ sampler_loop "$INTERVAL"
+ ) >> "$LOG_FILE" 2>&1 &
+
+ echo $! > "$PID_FILE"
+ echo "Memory sampler started (pid $(cat "$PID_FILE"), interval
${INTERVAL}s)"
+ echo "Samples: $SAMPLES_FILE"
+}
+
+cmd_stop() {
+ sampler_paths
+
+ if [ ! -f "$PID_FILE" ]; then
+ echo "Memory sampler is not running" >&2
+ cmd_summarize || true
+ return 0
+ fi
+
+ pid="$(cat "$PID_FILE" 2>/dev/null || true)"
+ touch "$STOP_FILE"
+
+ if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then
+ kill -TERM "$pid" 2>/dev/null || true
+ wait "$pid" 2>/dev/null || true
+ fi
+
+ rm -f "$PID_FILE" "$STOP_FILE"
+ echo "# stopped $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$LOG_FILE"
+ cmd_summarize || true
+ echo "Memory sampler stopped"
+}
+
+cmd_sample_once() {
+ sampler_paths
+ it_memory_write_samples_header "$SAMPLES_FILE"
+ it_memory_sample_once "$TARGET_DIR"
+}
+
+cmd_summarize() {
+ sampler_paths
+ if it_memory_summarize_samples "$SAMPLES_FILE" "$SUMMARY_FILE"; then
+ echo "Wrote $SUMMARY_FILE"
+ return 0
+ fi
+ echo "No memory samples to summarize in $SAMPLES_FILE" >&2
+ return 1
+}
+
+cmd_operator_note() {
+ if [ "$PRINT_ONLY" = true ]; then
+ it_generate_operator_note "$TARGET_DIR" "$SCRIPT_DIR"
+ return 0
+ fi
+ if it_write_operator_note_file "$TARGET_DIR" "$SCRIPT_DIR"; then
+ echo "Wrote $TARGET_DIR/$IT_OPERATOR_NOTE_FILE"
+ return 0
+ fi
+ echo "Could not generate operator note (missing $IT_RUN_TRACE in
$TARGET_DIR)" >&2
+ return 1
+}
+
+cmd_verify() {
+ it_memory_verify "$TARGET_DIR" "$SCRIPT_DIR"
+}
+
+main() {
+ parse_args "$@"
+ case "$COMMAND" in
+ start) cmd_start ;;
+ stop) cmd_stop ;;
+ sample-once) cmd_sample_once ;;
+ summarize) cmd_summarize ;;
+ operator-note) cmd_operator_note ;;
+ verify) cmd_verify ;;
+ *)
+ echo "Unknown command: $COMMAND" >&2
+ usage
+ ;;
+ esac
+}
+
+main "$@"
diff --git
a/services/src/test/java/org/apache/unomi/services/impl/cluster/ClusterServiceImplTest.java
b/services/src/test/java/org/apache/unomi/services/impl/cluster/ClusterServiceImplTest.java
index 435360e48..bd7bc231f 100644
---
a/services/src/test/java/org/apache/unomi/services/impl/cluster/ClusterServiceImplTest.java
+++
b/services/src/test/java/org/apache/unomi/services/impl/cluster/ClusterServiceImplTest.java
@@ -383,13 +383,14 @@ public class ClusterServiceImplTest {
persistenceService.save(recentNode);
});
- // Purge items older than cutoff (between old and recent)
- Date cutoff = new Date(System.currentTimeMillis() - 3L * 24 * 3600 *
1000); // 3 days ago
- clusterService.purge(cutoff);
+ // Purge and verify in system tenant — items were saved there;
load/purge are tenant-scoped.
+ executionContextManager.executeAsSystem(() -> {
+ Date cutoff = new Date(System.currentTimeMillis() - 3L * 24 * 3600
* 1000); // 3 days ago
+ clusterService.purge(cutoff);
- // Verify: old node removed, recent node remains
- assertNull(persistenceService.load("old-node", ClusterNode.class));
- assertNotNull(persistenceService.load("recent-node",
ClusterNode.class));
+ assertNull(persistenceService.load("old-node", ClusterNode.class));
+ assertNotNull(persistenceService.load("recent-node",
ClusterNode.class));
+ });
}
@Test
@@ -411,11 +412,11 @@ public class ClusterServiceImplTest {
persistenceService.save(otherNode);
});
- // Execute purge by scope
- clusterService.purge("testScope");
+ executionContextManager.executeAsSystem(() -> {
+ clusterService.purge("testScope");
- // Verify: scoped node removed, other node remains
- assertNull(persistenceService.load("scoped-node", ClusterNode.class));
- assertNotNull(persistenceService.load("other-node",
ClusterNode.class));
+ assertNull(persistenceService.load("scoped-node",
ClusterNode.class));
+ assertNotNull(persistenceService.load("other-node",
ClusterNode.class));
+ });
}
}
diff --git
a/services/src/test/java/org/apache/unomi/services/impl/scheduler/SchedulerServiceImplTest.java
b/services/src/test/java/org/apache/unomi/services/impl/scheduler/SchedulerServiceImplTest.java
index 7a31d62ab..593867e2a 100644
---
a/services/src/test/java/org/apache/unomi/services/impl/scheduler/SchedulerServiceImplTest.java
+++
b/services/src/test/java/org/apache/unomi/services/impl/scheduler/SchedulerServiceImplTest.java
@@ -2033,8 +2033,10 @@ public class SchedulerServiceImplTest {
taskId.set(systemTask.getItemId());
- // Wait for task to execute once
+ // Wait for task to execute once, then cancel immediately so
fixed-rate scheduling
+ // cannot fire a second execution before we assert the count (CI
timing flake).
assertTrue(executionLatch.await(TEST_TIMEOUT, TEST_TIME_UNIT), "Task
should execute");
+ schedulerService.cancelTask(systemTask.getItemId());
assertEquals(1, executionCount.get(), "Task should have executed
once");
// Force persistence to ensure the task is saved
@@ -2485,7 +2487,17 @@ public class SchedulerServiceImplTest {
@Test
@Tag("RecoveryTests")
public void testPreDestroyMarksStaleRunningPersistentTaskAsCrashed()
throws Exception {
- ScheduledTask runningTask = createTestTask("predestroy-crash-test",
ScheduledTask.TaskStatus.RUNNING);
+ // Persist once with a valid lock. createTestTask() saves RUNNING
tasks without a
+ // lock first, which lets the background recovery loop (initial delay
0) mark them
+ // CRASHED before preDestroy() under CI scheduling.
+ ScheduledTask runningTask = new ScheduledTask();
+ runningTask.setItemId(UUID.randomUUID().toString());
+ runningTask.setTaskType("predestroy-crash-test");
+ runningTask.setStatus(ScheduledTask.TaskStatus.RUNNING);
+ runningTask.setLockOwner(schedulerService.getNodeId());
+ runningTask.setLockDate(new Date());
+ runningTask.setExecutingNodeId(schedulerService.getNodeId());
+ persistenceService.save(runningTask);
persistenceService.refresh();
schedulerService.preDestroy();
