This is an automated email from the ASF dual-hosted git repository. rzo1 pushed a commit to branch reduce-distro-size in repository https://gitbox.apache.org/repos/asf/storm.git
commit 8cf5dd818ac46b1c2d857967e9d89a9ae502c550 Author: Richard Zowalla <[email protected]> AuthorDate: Tue Jun 30 19:45:35 2026 +0200 build: stop bundling storm-autocreds in the binary distribution storm-autocreds pulls in the full Hadoop/HBase client dependency tree (~79 MB, 43 jars unique to it) but is only needed on secure (Kerberos) clusters and is off by default. Ship only the README, consistent with the other external/* connectors, and add bin/storm-autocreds-fetch to retrieve the plugin and its runtime dependencies from Maven Central into extlib-daemon on demand. Also removes the now-unused storm-autocreds-bin assembly module. --- bin/storm-autocreds-fetch | 134 +++++++++++++++++++++ external/storm-autocreds/README.md | 101 ++++++++++++++++ .../final-package/src/main/assembly/binary.xml | 9 +- storm-dist/binary/pom.xml | 1 - storm-dist/binary/storm-autocreds-bin/pom.xml | 63 ---------- .../src/main/assembly/storm-autocreds.xml | 33 ----- storm-dist/binary/storm-kafka-monitor-bin/pom.xml | 69 ----------- .../src/main/assembly/storm-kafka-monitor.xml | 33 ----- 8 files changed, 241 insertions(+), 202 deletions(-) diff --git a/bin/storm-autocreds-fetch b/bin/storm-autocreds-fetch new file mode 100755 index 000000000..d585be72b --- /dev/null +++ b/bin/storm-autocreds-fetch @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Fetch the storm-autocreds plugin and its (Hadoop/HBase) runtime dependencies +# into the daemon classpath, so Nimbus/Supervisor can populate and renew HDFS +# and HBase delegation tokens on a secure (Kerberos) cluster. +# +# These jars are intentionally NOT bundled in the binary distribution to keep it +# small; only secure-Hadoop deployments need them. See +# external/storm-autocreds/README.md for details. + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: storm-autocreds-fetch [options] [-- <extra maven args>] + +Resolves org.apache.storm:storm-autocreds and its runtime dependencies from a +Maven repository (Maven Central by default) and copies them into the Storm +daemon classpath directory (extlib-daemon). + +Options: + --version <ver> Storm version to fetch (default: read from $STORM_HOME/RELEASE) + --dest <dir> Target directory (default: $STORM_HOME/extlib-daemon) + -h, --help Show this help + +Any arguments after "--" are passed through to Maven, e.g. to use an internal +mirror or an offline local repository: + storm-autocreds-fetch -- -s /path/settings.xml + storm-autocreds-fetch -- -Dmaven.repo.local=/path/to/offline-repo -o +EOF +} + +# Resolve symlinks so STORM_HOME is correct even when invoked via a link. +PRG="${0}" +while [ -h "${PRG}" ]; do + ls=$(ls -ld "${PRG}") + link=$(expr "${ls}" : '.*-> \(.*\)$') + if expr "${link}" : '/.*' > /dev/null; then + PRG="${link}" + else + PRG="$(dirname "${PRG}")/${link}" + fi +done +STORM_BIN_DIR=$(dirname "${PRG}") +STORM_HOME=$(cd "${STORM_BIN_DIR}/.." && pwd) + +VERSION="" +DEST="" +MVN_ARGS=() +while [ $# -gt 0 ]; do + case "${1}" in + --version) VERSION="${2}"; shift 2 ;; + --dest) DEST="${2}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + --) shift; MVN_ARGS=("$@"); break ;; + *) echo "Unknown option: ${1}" >&2; usage; exit 1 ;; + esac +done + +if [ -z "${VERSION}" ]; then + if [ -f "${STORM_HOME}/RELEASE" ]; then + VERSION=$(tr -d '[:space:]' < "${STORM_HOME}/RELEASE") + fi +fi +if [ -z "${VERSION}" ]; then + echo "Error: could not determine Storm version. Pass --version <ver>." >&2 + exit 1 +fi + +if [ -z "${DEST}" ]; then + DEST="${STORM_HOME}/extlib-daemon" +fi + +MVN="${MAVEN_HOME:+${MAVEN_HOME}/bin/}mvn" +if ! command -v "${MVN}" > /dev/null 2>&1; then + echo "Error: '${MVN}' not found on PATH. Install Apache Maven or set MAVEN_HOME." >&2 + exit 1 +fi + +mkdir -p "${DEST}" + +# Use a throwaway POM that depends on storm-autocreds; copy-dependencies then +# pulls the exact runtime closure (honoring the exclusions declared in the +# published storm-autocreds POM). storm-client is 'provided' there and is +# correctly skipped, since it already ships in lib/. +TMP_DIR=$(mktemp -d) +trap 'rm -rf "${TMP_DIR}"' EXIT +cat > "${TMP_DIR}/pom.xml" <<EOF +<project xmlns="http://maven.apache.org/POM/4.0.0";> + <modelVersion>4.0.0</modelVersion> + <groupId>org.apache.storm.tools</groupId> + <artifactId>storm-autocreds-fetch</artifactId> + <version>${VERSION}</version> + <packaging>pom</packaging> + <dependencies> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-autocreds</artifactId> + <version>${VERSION}</version> + </dependency> + </dependencies> +</project> +EOF + +echo "Fetching org.apache.storm:storm-autocreds:${VERSION} (runtime closure) into:" +echo " ${DEST}" +"${MVN}" -q -f "${TMP_DIR}/pom.xml" \ + org.apache.maven.plugins:maven-dependency-plugin:copy-dependencies \ + -DincludeScope=runtime \ + -DoutputDirectory="${DEST}" \ + ${MVN_ARGS[@]+"${MVN_ARGS[@]}"} + +echo "Done. ${DEST} now contains:" +ls -1 "${DEST}" | sed 's/^/ /' +echo +echo "Restart the Storm daemons (Nimbus, Supervisor) to pick up the new classpath," +echo "then configure the autocreds plugins in storm.yaml. See" +echo "external/storm-autocreds/README.md for the required settings." diff --git a/external/storm-autocreds/README.md b/external/storm-autocreds/README.md new file mode 100644 index 000000000..024593529 --- /dev/null +++ b/external/storm-autocreds/README.md @@ -0,0 +1,101 @@ +# Storm Auto Credentials (HDFS / HBase) + +`storm-autocreds` lets Storm automatically acquire, distribute and renew +**Hadoop delegation tokens** so that topologies can talk to a secure (Kerberos) +HDFS or HBase cluster without distributing keytabs to every worker host. + +* On topology submission, **Nimbus** obtains delegation tokens on behalf of the + submitting user and ships them with the topology. +* **Workers** unpack the tokens into their `Subject` / `UserGroupInformation`. +* **Nimbus** periodically renews the tokens for long-running topologies. + +See `docs/SECURITY.md` ("Automatic Credentials Push and Renewal") for the full +design. + +## Why the jars are not bundled + +Because these plugins run on the **daemon** classpath (Nimbus/Supervisor) and +pull in the full Hadoop and HBase client dependency trees, they are **not** +shipped inside the binary distribution — only secure-Hadoop deployments need +them, and bundling them would bloat the distribution for everyone. This is the +same convention used by the other `external/*` connectors. + +## Installing + +The plugins must be present on the **daemon** classpath, i.e. in +`$STORM_HOME/extlib-daemon` on Nimbus and the Supervisors. + +### Option 1 — use the helper script (recommended) + +The distribution ships a helper that resolves `storm-autocreds` and its runtime +dependencies from Maven Central and copies them into `extlib-daemon`: + +```bash +$STORM_HOME/bin/storm-autocreds-fetch +``` + +It detects the Storm version from `$STORM_HOME/RELEASE`. Useful options: + +```bash +# explicit version / target directory +bin/storm-autocreds-fetch --version 3.0.0 --dest /opt/storm/extlib-daemon + +# pass extra arguments through to Maven (internal mirror / offline repo) +bin/storm-autocreds-fetch -- -s /etc/maven/settings.xml +bin/storm-autocreds-fetch -- -Dmaven.repo.local=/srv/offline-repo -o +``` + +Maven must be available on the host running the script (it does not have to be +installed on the cluster nodes — you can run it once and copy the resulting jars +to every daemon host). + +### Option 2 — build from source + +```bash +mvn -pl external/storm-autocreds -am package +cp external/storm-autocreds/target/storm-autocreds-*.jar \ + $(find ~/.m2 -name 'hadoop-auth-*.jar' -o -name 'hbase-client-*.jar') \ + $STORM_HOME/extlib-daemon/ +``` + +(Prefer Option 1 — it resolves the complete, correct dependency closure for you.) + +Restart Nimbus and the Supervisors after adding the jars so the new classpath +takes effect. + +## Configuring + +Add the following to `storm.yaml`. The `*Nimbus` classes run on Nimbus (acquire +and renew tokens); the non-`Nimbus` classes run in the worker (unpack tokens). + +```yaml +# Worker side: unpack the tokens into the worker Subject. +topology.auto-credentials: + - org.apache.storm.hdfs.security.AutoHDFS + - org.apache.storm.hbase.security.AutoHBase + +# Nimbus side: obtain the tokens on behalf of the submitter. +nimbus.autocredential.plugins.classes: + - org.apache.storm.hdfs.security.AutoHDFSNimbus + - org.apache.storm.hbase.security.AutoHBaseNimbus + +# Nimbus side: renew the tokens for long-running topologies. +nimbus.credential.renewers.classes: + - org.apache.storm.hdfs.security.AutoHDFSNimbus + - org.apache.storm.hbase.security.AutoHBaseNimbus +``` + +Relevant credential settings: + +| Setting | Purpose | +|---|---| +| `hdfs.keytab.file` / `hdfs.kerberos.principal` | Nimbus principal used to fetch HDFS tokens | +| `hdfs.kerberos.principal` | HDFS service principal | +| `hbase.keytab.file` / `hbase.kerberos.principal` | Nimbus principal used to fetch HBase tokens | +| `topology.hdfs.uri` | NameNode URI (defaults to the cluster `fs.defaultFS`) | +| `hdfsCredentialsConfigKeys` / `hbaseCredentialsConfigKeys` | optional list of per-cluster config keys when talking to multiple clusters | + +Use only the HDFS or only the HBase entries if you need just one of them. + +For the full secure-cluster setup (Kerberos, impersonation, ACLs) see +`docs/SECURITY.md`. diff --git a/storm-dist/binary/final-package/src/main/assembly/binary.xml b/storm-dist/binary/final-package/src/main/assembly/binary.xml index ee5cb60e3..2f4afedc2 100644 --- a/storm-dist/binary/final-package/src/main/assembly/binary.xml +++ b/storm-dist/binary/final-package/src/main/assembly/binary.xml @@ -241,12 +241,15 @@ </includes> </fileSet> - <!-- Autocred plugins --> + <!-- Autocred plugins: ship only the README, consistent with the other + external connectors. The (Hadoop/HBase) jars are no longer bundled to + keep the distribution small; operators fetch them on demand with + bin/storm-autocreds-fetch (see external/storm-autocreds/README.md). --> <fileSet> - <directory>${project.basedir}/../storm-autocreds-bin/target/autocreds/autocreds/lib-autocreds</directory> + <directory>${project.basedir}/../../../external/storm-autocreds</directory> <outputDirectory>external/storm-autocreds</outputDirectory> <includes> - <include>*jar</include> + <include>README.*</include> </includes> </fileSet> <!-- storm-submit-tools --> diff --git a/storm-dist/binary/pom.xml b/storm-dist/binary/pom.xml index 94209592e..166ee9442 100644 --- a/storm-dist/binary/pom.xml +++ b/storm-dist/binary/pom.xml @@ -47,7 +47,6 @@ <modules> <module>storm-client-bin</module> <module>storm-webapp-bin</module> - <module>storm-autocreds-bin</module> <module>storm-submit-tools-bin</module> <module>storm-kafka-monitor-bin</module> <!-- Final package must be last, as it needs to copy files from the other modules --> diff --git a/storm-dist/binary/storm-autocreds-bin/pom.xml b/storm-dist/binary/storm-autocreds-bin/pom.xml deleted file mode 100644 index 8b42b902f..000000000 --- a/storm-dist/binary/storm-autocreds-bin/pom.xml +++ /dev/null @@ -1,63 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.storm</groupId> - <artifactId>apache-storm-bin</artifactId> - <version>3.0.0-SNAPSHOT</version> - </parent> - <artifactId>storm-autocreds-bin</artifactId> - <packaging>pom</packaging> - - - <name>Storm Autocreds Binary</name> - <dependencies> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>storm-autocreds</artifactId> - <version>${project.version}</version> - </dependency> - </dependencies> - - <build> - <finalName>autocreds</finalName> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-assembly-plugin</artifactId> - <executions> - <execution> - <phase>prepare-package</phase> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> - <configuration> - <attach>false</attach> - <runOnlyAtExecutionRoot>false</runOnlyAtExecutionRoot> - <descriptors> - <descriptor>${project.basedir}/src/main/assembly/storm-autocreds.xml</descriptor> - </descriptors> - <appendAssemblyId>false</appendAssemblyId> - </configuration> - </plugin> - </plugins> - </build> -</project> diff --git a/storm-dist/binary/storm-autocreds-bin/src/main/assembly/storm-autocreds.xml b/storm-dist/binary/storm-autocreds-bin/src/main/assembly/storm-autocreds.xml deleted file mode 100644 index 1cd914cb7..000000000 --- a/storm-dist/binary/storm-autocreds-bin/src/main/assembly/storm-autocreds.xml +++ /dev/null @@ -1,33 +0,0 @@ -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd";> - <id>storm-autocreds-bin</id> - <formats> - <format>dir</format> - </formats> - <dependencySets> - <dependencySet> - <useProjectArtifact>false</useProjectArtifact> - <outputDirectory>lib-autocreds</outputDirectory> - <unpack>false</unpack> - </dependencySet> - </dependencySets> -</assembly> diff --git a/storm-dist/binary/storm-kafka-monitor-bin/pom.xml b/storm-dist/binary/storm-kafka-monitor-bin/pom.xml deleted file mode 100644 index 4d9e569a9..000000000 --- a/storm-dist/binary/storm-kafka-monitor-bin/pom.xml +++ /dev/null @@ -1,69 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.storm</groupId> - <artifactId>apache-storm-bin</artifactId> - <version>3.0.0-SNAPSHOT</version> - </parent> - <artifactId>storm-kafka-monitor-bin</artifactId> - <packaging>pom</packaging> - - - <name>Storm Kafka Monitor Binary</name> - <dependencies> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>storm-kafka-monitor</artifactId> - <version>${project.version}</version> - </dependency> - <!-- Explicitly included as dependencyManagement sets this to provided scope --> - <dependency> - <groupId>org.apache.kafka</groupId> - <artifactId>kafka-clients</artifactId> - <scope>compile</scope> - </dependency> - </dependencies> - - <build> - <finalName>kafka-monitor</finalName> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-assembly-plugin</artifactId> - <executions> - <execution> - <phase>prepare-package</phase> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> - <configuration> - <attach>false</attach> - <runOnlyAtExecutionRoot>false</runOnlyAtExecutionRoot> - <descriptors> - <descriptor>${project.basedir}/src/main/assembly/storm-kafka-monitor.xml</descriptor> - </descriptors> - <appendAssemblyId>false</appendAssemblyId> - </configuration> - </plugin> - </plugins> - </build> -</project> diff --git a/storm-dist/binary/storm-kafka-monitor-bin/src/main/assembly/storm-kafka-monitor.xml b/storm-dist/binary/storm-kafka-monitor-bin/src/main/assembly/storm-kafka-monitor.xml deleted file mode 100644 index 021b472c8..000000000 --- a/storm-dist/binary/storm-kafka-monitor-bin/src/main/assembly/storm-kafka-monitor.xml +++ /dev/null @@ -1,33 +0,0 @@ -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd";> - <id>storm-kafka-monitor-bin</id> - <formats> - <format>dir</format> - </formats> - <dependencySets> - <dependencySet> - <useProjectArtifact>false</useProjectArtifact> - <outputDirectory>lib-kafka-monitor</outputDirectory> - <unpack>false</unpack> - </dependencySet> - </dependencySets> -</assembly>
