This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/storm.git
The following commit(s) were added to refs/heads/master by this push:
new 2f0a63a2a Fix worker launch: add lib-common to the supervisor's worker
classpath and restore the lib-worker drop-in directory (#8905)
2f0a63a2a is described below
commit 2f0a63a2ac0cb5934901e485bfcf9256362181c4
Author: reiabreu <[email protected]>
AuthorDate: Sun Jul 19 11:26:38 2026 +0100
Fix worker launch: add lib-common to the supervisor's worker classpath and
restore the lib-worker drop-in directory (#8905)
* Fix supervisor-launched workers: add lib-common to the worker framework
classpath
Since the lib de-duplication (#8819) the distribution ships the worker jars
in lib-common instead of lib-worker. bin/storm.py was updated, but
BasicContainer.frameworkClasspath() still built the worker launch classpath
from lib-worker only, so supervisor-launched workers crashed with
ClassNotFoundException: org.apache.storm.LogWriter.
Add lib-common to the framework classpath, keeping lib-worker as the
drop-in location for worker-only jars.
* test: cover the real frameworkClasspath() in BasicContainerTest
The existing launch tests mock frameworkClasspath() away (FRAMEWORK_CP),
which is how the lib-common regression slipped through. Exercise the real
implementation via a hook in MockBasicContainer and assert that both
lib-common/* and lib-worker/* are on the worker classpath.
* build: ship an empty lib-worker drop-in directory in the distribution
The lib de-duplication removed lib-worker entirely, but both bin/storm.py
and BasicContainer keep lib-worker/* on the worker classpath, and operators
use the directory as the drop-in point for worker-only jars. Restore it as
an empty directory holding only a README that documents the new
lib / lib-common / lib-worker layout, in both the full and lite
distributions.
---------
Co-authored-by: Richard Zowalla <[email protected]>
---
.../final-package/src/main/assembly/common.xml | 6 +++
.../src/main/dist/lib-worker/README.md | 15 +++++++
.../storm/daemon/supervisor/BasicContainer.java | 5 +++
.../daemon/supervisor/BasicContainerTest.java | 47 ++++++++++++++++++++++
4 files changed, 73 insertions(+)
diff --git a/storm-dist/binary/final-package/src/main/assembly/common.xml
b/storm-dist/binary/final-package/src/main/assembly/common.xml
index 6b50cc8ad..402fd3837 100644
--- a/storm-dist/binary/final-package/src/main/assembly/common.xml
+++ b/storm-dist/binary/final-package/src/main/assembly/common.xml
@@ -38,6 +38,12 @@
<include>*.jar</include>
</includes>
</fileSet>
+ <!-- lib-worker ships empty (README only): it stays on the worker
classpath as the
+ drop-in point for worker-only jars, matching the pre-dedup 2.x
layout. -->
+ <fileSet>
+ <directory>${project.basedir}/src/main/dist/lib-worker</directory>
+ <outputDirectory>lib-worker</outputDirectory>
+ </fileSet>
<fileSet>
<directory>${project.basedir}/../storm-webapp-bin/target/webapp/webapp/</directory>
<outputDirectory>.</outputDirectory>
diff --git a/storm-dist/binary/final-package/src/main/dist/lib-worker/README.md
b/storm-dist/binary/final-package/src/main/dist/lib-worker/README.md
new file mode 100644
index 000000000..65f2d6a5a
--- /dev/null
+++ b/storm-dist/binary/final-package/src/main/dist/lib-worker/README.md
@@ -0,0 +1,15 @@
+# lib-worker
+
+Drop-in directory for worker-only jars.
+
+Jars placed here are added to the classpath of every worker JVM launched by
+the supervisor, but not to the daemon (nimbus / supervisor / ui) classpath.
+
+The distribution itself ships no worker-only jars: the jars shared by the
+daemons and the workers live in `lib-common/`, daemon-only jars in `lib/`.
+The classpaths are composed as:
+
+ daemon classpath = lib-common + lib
+ worker classpath = lib-common + lib-worker
+
+See also `extlib/`, which is added to both the daemon and worker classpaths.
diff --git
a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java
b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java
index 7583a96db..35ce6c56f 100644
---
a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java
+++
b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java
@@ -372,6 +372,10 @@ public class BasicContainer extends Container {
}
protected List<String> frameworkClasspath(SimpleVersion topoVersion) {
+ // Jars shared by the daemon and worker classpaths are de-duplicated
into lib-common
+ // (see storm-dist dedup-libs.py); storm-client and its dependencies
live there, so the
+ // worker classpath needs lib-common in addition to lib-worker.
+ File stormCommonLibDir = new File(stormHome, "lib-common");
File stormWorkerLibDir = new File(stormHome, "lib-worker");
String topoConfDir = System.getenv("STORM_CONF_DIR") != null
? System.getenv("STORM_CONF_DIR")
@@ -379,6 +383,7 @@ public class BasicContainer extends Container {
File stormExtlibDir = new File(stormHome, "extlib");
String extcp = System.getenv("STORM_EXT_CLASSPATH");
List<String> pathElements = new LinkedList<>();
+ pathElements.add(getWildcardDir(stormCommonLibDir));
pathElements.add(getWildcardDir(stormWorkerLibDir));
pathElements.add(getWildcardDir(stormExtlibDir));
pathElements.add(extcp);
diff --git
a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java
b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java
index a4f6ec93a..4f941d42e 100644
---
a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java
+++
b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java
@@ -31,11 +31,13 @@ import org.apache.storm.generated.StormTopology;
import org.apache.storm.utils.LocalState;
import org.apache.storm.utils.SimpleVersion;
import org.apache.storm.utils.Utils;
+import org.apache.storm.utils.VersionInfo;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
@@ -402,6 +404,47 @@ public class BasicContainerTest {
"storm.log.dir", stormLogDir);
}
+ @Test
+ public void testFrameworkClasspathIncludesSharedAndWorkerLibs() throws
Exception {
+ final String topoId = "test_topology_classpath";
+ final int supervisorPort = 6628;
+ final int port = 8080;
+ final String stormHome = ContainerTest.asAbsPath("tmp", "storm-home");
+ final String stormLogDir = ContainerTest.asFile(".",
"target").getCanonicalPath();
+ final String stormLocal = ContainerTest.asAbsPath("tmp",
"storm-local");
+
+ final Map<String, Object> superConf = new HashMap<>();
+ superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
+ superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
+
+ LocalAssignment la = new LocalAssignment();
+ la.set_topology_id(topoId);
+
+ AdvancedFSOps ops = mock(AdvancedFSOps.class);
+ when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+
+ LocalState ls = mock(LocalState.class);
+ MockResourceIsolationManager iso = new MockResourceIsolationManager();
+
+ checkpoint(() -> {
+ MockBasicContainer mc = new
MockBasicContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", supervisorPort, port, la, iso, ls,
"worker-id", new StormMetricsRegistry(),
+ new HashMap<>(), ops, "profile");
+
+ List<String> cp =
mc.realFrameworkClasspath(VersionInfo.OUR_VERSION);
+
+ // The distribution de-duplicates the jars shared by
the daemon and worker
+ // classpaths into lib-common; storm-client (LogWriter,
Worker) ships there,
+ // so a worker launched without lib-common on the
classpath cannot start.
+ String libCommon = stormHome + File.separator +
"lib-common" + File.separator + "*";
+ String libWorker = stormHome + File.separator +
"lib-worker" + File.separator + "*";
+ assertTrue(cp.contains(libCommon), "worker classpath
must include lib-common/*, got: " + cp);
+ assertTrue(cp.contains(libWorker), "worker classpath
must include lib-worker/*, got: " + cp);
+ },
+ ConfigUtils.STORM_HOME, stormHome,
+ "storm.log.dir", stormLogDir);
+ }
+
@Test
public void testLaunchStorm1version() throws Exception {
final String topoId = "test_topology_storm_1.x";
@@ -712,6 +755,10 @@ public class BasicContainerTest {
return Collections.singletonList("FRAMEWORK_CP");
}
+ public List<String> realFrameworkClasspath(SimpleVersion version) {
+ return super.frameworkClasspath(version);
+ }
+
@Override
protected String javaLibraryPath(String stormRoot, Map<String, Object>
conf) {
return "JLP";
