This is an automated email from the ASF dual-hosted git repository. rzo1 pushed a commit to branch reduce-distro-size in repository https://gitbox.apache.org/repos/asf/storm.git
commit 696dbc7e0020c6a8c9e20a1c6ec97012ce20d200 Author: Richard Zowalla <[email protected]> AuthorDate: Tue Jun 30 19:58:51 2026 +0200 build: stop bundling storm-kafka-monitor in the binary distribution The storm-kafka-monitor jars (and their Kafka client dependencies, ~38 MB) are only needed to display Kafka spout lag in the UI or to run the bin/storm-kafka-monitor command. Ship only the README, consistent with the other external/* connectors, and add bin/storm-kafka-monitor-fetch to retrieve the tool and its runtime dependencies from Maven Central into lib-tools/storm-kafka-monitor on demand. Guard the UI against the jars being absent: TopologySpoutLag now detects whether storm-kafka-monitor is installed and, when it is not, surfaces an actionable message (and logs it once) instead of failing the lag shell-out. The bin/storm-kafka-monitor wrapper prints the same hint instead of a ClassNotFound error. Also removes the now-unused storm-kafka-monitor-bin assembly module. --- bin/storm-kafka-monitor | 9 +- bin/storm-kafka-monitor-fetch | 133 +++++++++++++++++++++ external/storm-kafka-monitor/README.md | 24 ++++ .../org/apache/storm/utils/TopologySpoutLag.java | 36 +++++- .../final-package/src/main/assembly/binary.xml | 11 +- storm-dist/binary/pom.xml | 1 - 6 files changed, 207 insertions(+), 7 deletions(-) diff --git a/bin/storm-kafka-monitor b/bin/storm-kafka-monitor index 9bd11054c..a586c1d8d 100755 --- a/bin/storm-kafka-monitor +++ b/bin/storm-kafka-monitor @@ -49,4 +49,11 @@ if [ -z "$JAVA_HOME" ]; then else JAVA="$JAVA_HOME/bin/java" fi -exec $JAVA $STORM_JAAS_CONF_PARAM $STORM_JAR_JVM_OPTS -cp "$STORM_BASE_DIR/lib-tools/storm-kafka-monitor/*" org.apache.storm.kafka.monitor.KafkaOffsetLagUtil "$@" +# The storm-kafka-monitor jars are not bundled in the distribution; they are fetched on demand. +KAFKA_MONITOR_LIB="$STORM_BASE_DIR/lib-tools/storm-kafka-monitor" +if ! ls "$KAFKA_MONITOR_LIB"/*.jar >/dev/null 2>&1; then + echo "storm-kafka-monitor is not installed (no jars in $KAFKA_MONITOR_LIB)." >&2 + echo "Run '$STORM_BIN_DIR/storm-kafka-monitor-fetch' to download it, then retry." >&2 + exit 1 +fi +exec $JAVA $STORM_JAAS_CONF_PARAM $STORM_JAR_JVM_OPTS -cp "$KAFKA_MONITOR_LIB/*" org.apache.storm.kafka.monitor.KafkaOffsetLagUtil "$@" diff --git a/bin/storm-kafka-monitor-fetch b/bin/storm-kafka-monitor-fetch new file mode 100755 index 000000000..573312d87 --- /dev/null +++ b/bin/storm-kafka-monitor-fetch @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Fetch the storm-kafka-monitor tool and its (Kafka client) runtime dependencies +# into lib-tools/storm-kafka-monitor, enabling the "Kafka spout lag" display in +# the Storm UI and the bin/storm-kafka-monitor command. +# +# These jars are intentionally NOT bundled in the binary distribution to keep it +# small; they are only needed when running Kafka spouts and wanting lag info. The +# UI degrades gracefully when they are absent. See +# external/storm-kafka-monitor/README.md for details. + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: storm-kafka-monitor-fetch [options] [-- <extra maven args>] + +Resolves org.apache.storm:storm-kafka-monitor and its runtime dependencies from +a Maven repository (Maven Central by default) and copies them into +lib-tools/storm-kafka-monitor. + +Options: + --version <ver> Storm version to fetch (default: read from $STORM_HOME/RELEASE) + --dest <dir> Target directory (default: $STORM_HOME/lib-tools/storm-kafka-monitor) + -h, --help Show this help + +Any arguments after "--" are passed through to Maven, e.g. to use an internal +mirror or an offline local repository: + storm-kafka-monitor-fetch -- -s /path/settings.xml + storm-kafka-monitor-fetch -- -Dmaven.repo.local=/path/to/offline-repo -o +EOF +} + +# Resolve symlinks so STORM_HOME is correct even when invoked via a link. +PRG="${0}" +while [ -h "${PRG}" ]; do + ls=$(ls -ld "${PRG}") + link=$(expr "${ls}" : '.*-> \(.*\)$') + if expr "${link}" : '/.*' > /dev/null; then + PRG="${link}" + else + PRG="$(dirname "${PRG}")/${link}" + fi +done +STORM_BIN_DIR=$(dirname "${PRG}") +STORM_HOME=$(cd "${STORM_BIN_DIR}/.." && pwd) + +VERSION="" +DEST="" +MVN_ARGS=() +while [ $# -gt 0 ]; do + case "${1}" in + --version) VERSION="${2}"; shift 2 ;; + --dest) DEST="${2}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + --) shift; MVN_ARGS=("$@"); break ;; + *) echo "Unknown option: ${1}" >&2; usage; exit 1 ;; + esac +done + +if [ -z "${VERSION}" ]; then + if [ -f "${STORM_HOME}/RELEASE" ]; then + VERSION=$(tr -d '[:space:]' < "${STORM_HOME}/RELEASE") + fi +fi +if [ -z "${VERSION}" ]; then + echo "Error: could not determine Storm version. Pass --version <ver>." >&2 + exit 1 +fi + +if [ -z "${DEST}" ]; then + DEST="${STORM_HOME}/lib-tools/storm-kafka-monitor" +fi + +MVN="${MAVEN_HOME:+${MAVEN_HOME}/bin/}mvn" +if ! command -v "${MVN}" > /dev/null 2>&1; then + echo "Error: '${MVN}' not found on PATH. Install Apache Maven or set MAVEN_HOME." >&2 + exit 1 +fi + +mkdir -p "${DEST}" + +# Use a throwaway POM that depends on storm-kafka-monitor; copy-dependencies then +# pulls the exact runtime closure. The artifact itself is a direct dependency and +# is therefore copied too. +TMP_DIR=$(mktemp -d) +trap 'rm -rf "${TMP_DIR}"' EXIT +cat > "${TMP_DIR}/pom.xml" <<EOF +<project xmlns="http://maven.apache.org/POM/4.0.0";> + <modelVersion>4.0.0</modelVersion> + <groupId>org.apache.storm.tools</groupId> + <artifactId>storm-kafka-monitor-fetch</artifactId> + <version>${VERSION}</version> + <packaging>pom</packaging> + <dependencies> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-kafka-monitor</artifactId> + <version>${VERSION}</version> + </dependency> + </dependencies> +</project> +EOF + +echo "Fetching org.apache.storm:storm-kafka-monitor:${VERSION} (runtime closure) into:" +echo " ${DEST}" +"${MVN}" -q -f "${TMP_DIR}/pom.xml" \ + org.apache.maven.plugins:maven-dependency-plugin:copy-dependencies \ + -DincludeScope=runtime \ + -DoutputDirectory="${DEST}" \ + ${MVN_ARGS[@]+"${MVN_ARGS[@]}"} + +echo "Done. ${DEST} now contains:" +ls -1 "${DEST}" | sed 's/^/ /' +echo +echo "Restart the Storm UI to enable Kafka spout lag display, or run" +echo "bin/storm-kafka-monitor directly. See external/storm-kafka-monitor/README.md." diff --git a/external/storm-kafka-monitor/README.md b/external/storm-kafka-monitor/README.md index a483f4bde..8e5bd37ef 100644 --- a/external/storm-kafka-monitor/README.md +++ b/external/storm-kafka-monitor/README.md @@ -2,6 +2,30 @@ Tool to query kafka spout lags and show in Storm UI +## Installation + +The storm-kafka-monitor jars (and their Kafka client dependencies) are **not** +bundled in the binary distribution to keep it small — they are only needed to +display Kafka spout lag in the UI or to run the `storm-kafka-monitor` command. +The Storm UI degrades gracefully when they are absent (no lag is shown and a +hint is logged once). + +To enable it, install the jars on the UI host with the helper script, then +restart the UI: + +```bash +$STORM_HOME/bin/storm-kafka-monitor-fetch +``` + +It resolves `org.apache.storm:storm-kafka-monitor` and its runtime dependencies +from Maven Central into `lib-tools/storm-kafka-monitor`. Useful options: + +```bash +bin/storm-kafka-monitor-fetch --version 3.0.0 +# pass extra arguments through to Maven (internal mirror / offline repo) +bin/storm-kafka-monitor-fetch -- -s /etc/maven/settings.xml +``` + ## Usage This tool provides a way to query kafka offsets that the spout has consumed successfully and the latest offsets in kafka. It provides an easy way to see how the topology is performing. It is a command line diff --git a/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java b/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java index 7d8d7bbc8..a230e1e7a 100644 --- a/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java +++ b/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java @@ -54,6 +54,11 @@ public class TopologySpoutLag { BOOTSTRAP_CONFIG, SECURITY_PROTOCOL_CONFIG)); private static final Logger LOGGER = LoggerFactory.getLogger(TopologySpoutLag.class); + // The storm-kafka-monitor jars are not bundled in the binary distribution; operators install them + // on demand (bin/storm-kafka-monitor-fetch). Log the "not installed" hint at most once to avoid + // spamming the UI logs, which poll the lag endpoint periodically. + private static volatile boolean warnedMonitorMissing = false; + public static Map<String, Map<String, Object>> lag(StormTopology stormTopology, Map<String, Object> topologyConf) { Map<String, Map<String, Object>> result = new HashMap<>(); Map<String, SpoutSpec> spouts = stormTopology.get_spouts(); @@ -69,6 +74,24 @@ public class TopologySpoutLag { return result; } + /** + * Checks whether the storm-kafka-monitor jars (invoked by bin/storm-kafka-monitor) are present. + * They are not bundled in the binary distribution and are fetched on demand, so the UI must + * degrade gracefully when they are absent rather than failing the lag shell-out. + * + * @return true if the monitor appears installed, or if STORM_BASE_DIR is unknown (in which case + * the legacy behavior of attempting the shell-out is preserved). + */ + private static boolean isKafkaMonitorInstalled() { + String stormHomeDir = System.getenv("STORM_BASE_DIR"); + if (stormHomeDir == null) { + return true; + } + File libDir = new File(new File(stormHomeDir, "lib-tools"), "storm-kafka-monitor"); + File[] jars = libDir.listFiles((dir, name) -> name.endsWith(".jar")); + return jars != null && jars.length > 0; + } + private static List<String> getCommandLineOptionsForNewKafkaSpout(Map<String, Object> jsonConf) { LOGGER.debug("json configuration: {}", jsonConf); @@ -166,7 +189,18 @@ public class TopologySpoutLag { LOGGER.debug("Command to run: {}", commands); // if commands contains one or more null value, spout is compiled with lower version of storm-kafka-client - if (!commands.contains(null)) { + if (!commands.contains(null) && !isKafkaMonitorInstalled()) { + errorMsg = "Kafka spout lag monitoring is unavailable because the storm-kafka-monitor " + + "jars are not installed. They are no longer bundled in the binary distribution; " + + "run 'bin/storm-kafka-monitor-fetch' on the UI host (and restart the UI) to enable it."; + if (!warnedMonitorMissing) { + warnedMonitorMissing = true; + LOGGER.info(errorMsg); + } + if (extraPropertiesFile != null) { + extraPropertiesFile.delete(); + } + } else if (!commands.contains(null)) { try { String resultFromMonitor = new ShellCommandRunnerImpl().execCommand(commands.toArray(new String[0])); diff --git a/storm-dist/binary/final-package/src/main/assembly/binary.xml b/storm-dist/binary/final-package/src/main/assembly/binary.xml index 2f4afedc2..e41226110 100644 --- a/storm-dist/binary/final-package/src/main/assembly/binary.xml +++ b/storm-dist/binary/final-package/src/main/assembly/binary.xml @@ -232,12 +232,15 @@ <include></include> </includes> </fileSet> - <!-- $STORM_HOME/toollib --> + <!-- storm-kafka-monitor: ship only the README. The (Kafka client) jars are no longer bundled + to keep the distribution small; they are only needed to display Kafka spout lag in the UI. + Operators fetch them on demand with bin/storm-kafka-monitor-fetch (see + external/storm-kafka-monitor/README.md). The UI degrades gracefully when they are absent. --> <fileSet> - <directory>${project.basedir}/../storm-kafka-monitor-bin/target/kafka-monitor/kafka-monitor/lib-kafka-monitor</directory> - <outputDirectory>lib-tools/storm-kafka-monitor</outputDirectory> + <directory>${project.basedir}/../../../external/storm-kafka-monitor</directory> + <outputDirectory>external/storm-kafka-monitor</outputDirectory> <includes> - <include>*jar</include> + <include>README.*</include> </includes> </fileSet> diff --git a/storm-dist/binary/pom.xml b/storm-dist/binary/pom.xml index 166ee9442..3b58038d1 100644 --- a/storm-dist/binary/pom.xml +++ b/storm-dist/binary/pom.xml @@ -48,7 +48,6 @@ <module>storm-client-bin</module> <module>storm-webapp-bin</module> <module>storm-submit-tools-bin</module> - <module>storm-kafka-monitor-bin</module> <!-- Final package must be last, as it needs to copy files from the other modules --> <module>final-package</module> </modules>
