This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 2c7a75714f fix deadlock at high concurrency, fixes #8455 (#8456)
2c7a75714f is described below
commit 2c7a75714f19c55beb01c4042595e1c15fc9b1b4
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri Sep 18 18:35:59 2026 +0200
fix deadlock at high concurrency, fixes #8455 (#8456)
---
.../org/apache/hop/core/vfs/HopVfsNamespace.java | 43 ++-
.../org/apache/hop/core/vfs/HopVfsNamespaces.java | 148 ++++++--
.../hop/core/vfs/HopVfsNamespaceLockingTest.java | 401 +++++++++++++++++++++
integration-tests/hop_server/0019-child-delay.hpl | 108 ++++++
integration-tests/hop_server/0019-child-delay.hwf | 99 +++++
.../hop_server/0019-drive-200-workflows.hpl | 181 ++++++++++
.../main-0019-concurrent-exports-single-server.hwf | 168 +++++++++
7 files changed, 1108 insertions(+), 40 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/vfs/HopVfsNamespace.java
b/core/src/main/java/org/apache/hop/core/vfs/HopVfsNamespace.java
index 29ef06748c..303795db74 100644
--- a/core/src/main/java/org/apache/hop/core/vfs/HopVfsNamespace.java
+++ b/core/src/main/java/org/apache/hop/core/vfs/HopVfsNamespace.java
@@ -21,6 +21,8 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.logging.LogChannel;
@@ -56,6 +58,19 @@ public class HopVfsNamespace implements AutoCloseable {
/** How many executions are using this namespace; it closes when the last
one lets go. */
private int useCount;
+ /**
+ * Counted down once the named connections are registered. The namespace is
published to the
+ * registry before that happens, so the registering thread finds it when
reading metadata lands
+ * back in resolution; anybody else acquiring it waits here - outside every
lock - rather than
+ * resolving files before the connections are in.
+ */
+ private final CountDownLatch ready = new CountDownLatch(1);
+
+ /**
+ * How long a second user waits for a registration that is still going; see
{@link #awaitReady}.
+ */
+ private static final long READY_TIMEOUT_SECONDS = 300;
+
HopVfsNamespace(String description) throws HopException {
this.description = description;
this.fileSystemManager = HopVfs.createFileSystemManager();
@@ -100,7 +115,7 @@ public class HopVfsNamespace implements AutoCloseable {
*
* @throws HopException if the new manager cannot be created
*/
- void rebuild() throws HopException {
+ synchronized void rebuild() throws HopException {
DefaultFileSystemManager previous = fileSystemManager;
DefaultFileSystemManager fresh = HopVfs.createFileSystemManager();
try {
@@ -131,6 +146,32 @@ public class HopVfsNamespace implements AutoCloseable {
fileSystemManager.freeUnusedResources();
}
+ /** The named connections are in; see {@link #awaitReady()}. */
+ void markReady() {
+ ready.countDown();
+ }
+
+ /**
+ * Wait until whoever created this namespace has registered its named
connections. Returns at once
+ * for the creating thread itself and for anyone arriving afterwards; only a
second acquirer that
+ * overtakes the registration ever waits, and never for longer than the
registration takes.
+ * Bounded all the same: a registration that never finishes must not take
its waiters with it.
+ */
+ void awaitReady() {
+ try {
+ if (!ready.await(READY_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
+ LogChannel.GENERAL.logError(
+ "The named VFS connections of "
+ + description
+ + " were still being registered after "
+ + READY_TIMEOUT_SECONDS
+ + "s; resolving files without waiting for them");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
int retain() {
return ++useCount;
}
diff --git a/core/src/main/java/org/apache/hop/core/vfs/HopVfsNamespaces.java
b/core/src/main/java/org/apache/hop/core/vfs/HopVfsNamespaces.java
index 71e3adae1e..59c79b6ced 100644
--- a/core/src/main/java/org/apache/hop/core/vfs/HopVfsNamespaces.java
+++ b/core/src/main/java/org/apache/hop/core/vfs/HopVfsNamespaces.java
@@ -17,7 +17,9 @@
package org.apache.hop.core.vfs;
+import java.util.ArrayList;
import java.util.IdentityHashMap;
+import java.util.List;
import java.util.Map;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.logging.LogChannel;
@@ -39,6 +41,16 @@ import org.apache.hop.metadata.util.HopMetadataInstance;
* running a pipeline by filename.
*
* <p>See Apache Hop issue #8106.
+ *
+ * <h2>Locking</h2>
+ *
+ * <p>Registering the named connections of a namespace runs VFS plugin code,
which reads metadata,
+ * which resolves files through {@link HopVfs} - whose monitor another
execution may be holding
+ * while it looks its own namespace up here. So nothing in this class holds a
lock while calling
+ * out: the registry is only ever locked for a lookup or an update of the map,
never around plugin
+ * code or around closing a manager. A namespace created by one thread and
wanted by another is
+ * waited for through {@link HopVfsNamespace#awaitReady()}, again without a
lock. A Hop Server
+ * running two exports at once deadlocked before this was the rule.
*/
public class HopVfsNamespaces {
@@ -49,6 +61,13 @@ public class HopVfsNamespaces {
private static final Map<IHopMetadataProvider, HopVfsNamespace> NAMESPACES =
new IdentityHashMap<>();
+ /**
+ * Guards {@link #NAMESPACES} and the use counts. Held for map operations
only: whoever holds the
+ * {@link HopVfs} monitor may be waiting on it, so no plugin code, no
metadata reads and no
+ * closing of managers ever happen under it.
+ */
+ private static final Object REGISTRY = new Object();
+
/**
* Where the current namespace lives, for the call sites that resolve a file
without any variables
* in hand. Per thread unless something replaces it: {@code Pipeline} and
{@code Workflow} spawn
@@ -59,7 +78,7 @@ public class HopVfsNamespaces {
* <p>This is the fallback. Where an {@link IVariables} is available,
resolving through it is
* exact and does not depend on where the work ended up running.
*/
- private static IHopScope<HopVfsNamespace> scope =
IHopScope.inheritedByThreads();
+ private static volatile IHopScope<HopVfsNamespace> scope =
IHopScope.inheritedByThreads();
/**
* Change how far "current" reaches. Call this once at startup, before
anything resolves a file:
@@ -69,7 +88,7 @@ public class HopVfsNamespaces {
*
* @param newScope the scope to hold the current namespace in, null restores
the default
*/
- public static synchronized void setScope(IHopScope<HopVfsNamespace>
newScope) {
+ public static void setScope(IHopScope<HopVfsNamespace> newScope) {
scope = newScope == null ? IHopScope.inheritedByThreads() : newScope;
// A runtime only installs a scope of its own when several tenants share
the JVM. There, no
// tenant may fall back to the process wide manager: they would share it,
and one of them
@@ -78,7 +97,7 @@ public class HopVfsNamespaces {
}
/** Set when the runtime serves several tenants at once, so nobody shares
the process manager. */
- private static boolean isolateEverything;
+ private static volatile boolean isolateEverything;
/**
* Does this JVM serve several tenants at once - Hop Web, with a session
scope installed?
@@ -89,7 +108,7 @@ public class HopVfsNamespaces {
*
* @return true when a runtime installed a scope of its own with {@link
#setScope(IHopScope)}
*/
- public static synchronized boolean isIsolated() {
+ public static boolean isIsolated() {
return isolateEverything;
}
@@ -147,7 +166,7 @@ public class HopVfsNamespaces {
* @param description what this namespace belongs to, for logging
* @return the namespace, or null when the process wide manager applies
*/
- public static synchronized HopVfsNamespace acquire(
+ public static HopVfsNamespace acquire(
IVariables variables, IHopMetadataProvider metadataProvider, String
description) {
if (metadataProvider == null || isProcessMetadata(metadataProvider)) {
LogChannel.GENERAL.logDebug(
@@ -157,29 +176,64 @@ public class HopVfsNamespaces {
return null;
}
- HopVfsNamespace namespace = NAMESPACES.get(metadataProvider);
- if (namespace == null) {
+ // Publish-or-retry: the namespace is built outside the lock, so whoever
publishes first wins
+ // and a loser closes what it built. Nothing is ever put in the map or
counted that is not
+ // fully constructed, and a namespace released and removed between two
looks is simply not
+ // found the second time.
+ HopVfsNamespace namespace = null;
+ HopVfsNamespace fresh = null;
+ boolean creator = false;
+ while (namespace == null) {
+ synchronized (REGISTRY) {
+ namespace = NAMESPACES.get(metadataProvider);
+ if (namespace == null && fresh != null) {
+ // Publish before registering: registering the named connections
reads metadata, which
+ // resolves files, which lands right back in resolve() on this very
thread.
+ namespace = fresh;
+ fresh = null;
+ NAMESPACES.put(metadataProvider, namespace);
+ creator = true;
+ }
+ if (namespace != null) {
+ namespace.retain();
+ }
+ }
+ if (namespace == null) {
+ try {
+ fresh = new HopVfsNamespace(description);
+ } catch (HopException e) {
+ LogChannel.GENERAL.logError(
+ "Unable to create a VFS namespace for "
+ + description
+ + ", falling back to the process wide file system manager",
+ e);
+ return null;
+ }
+ }
+ }
+ if (fresh != null) {
+ // Somebody published one for this metadata in the meantime; theirs is
the namespace.
+ fresh.close();
+ }
+
+ if (creator) {
+ // Outside the lock: this runs plugin code that reads metadata through
HopVfs, and another
+ // execution may be holding the HopVfs monitor while it waits for a
lookup here.
try {
- namespace = new HopVfsNamespace(description);
- } catch (HopException e) {
- LogChannel.GENERAL.logError(
- "Unable to create a VFS namespace for "
+ namespace.registerNamedProviders(variables, metadataProvider);
+ LogChannel.GENERAL.logBasic(
+ "Created a VFS namespace for "
+ description
- + ", falling back to the process wide file system manager",
- e);
- return null;
+ + ", holding the named VFS connections of "
+ + metadataProvider.getDescription());
+ } finally {
+ namespace.markReady();
}
- // Publish before registering: registering the named connections reads
metadata, which
- // resolves files, which lands right back in resolve().
- NAMESPACES.put(metadataProvider, namespace);
- namespace.registerNamedProviders(variables, metadataProvider);
- LogChannel.GENERAL.logBasic(
- "Created a VFS namespace for "
- + description
- + ", holding the named VFS connections of "
- + metadataProvider.getDescription());
+ } else {
+ // Somebody else is still registering its connections; a file resolved
before they are in
+ // would silently miss a named connection.
+ namespace.awaitReady();
}
- namespace.retain();
return namespace;
}
@@ -190,17 +244,24 @@ public class HopVfsNamespaces {
*
* @param metadataProvider the metadata the namespace was taken for
*/
- public static synchronized void release(IHopMetadataProvider
metadataProvider) {
+ public static void release(IHopMetadataProvider metadataProvider) {
if (metadataProvider == null) {
return;
}
- HopVfsNamespace namespace = NAMESPACES.get(metadataProvider);
- if (namespace == null) {
- return;
+ HopVfsNamespace lastUse = null;
+ synchronized (REGISTRY) {
+ HopVfsNamespace namespace = NAMESPACES.get(metadataProvider);
+ if (namespace == null) {
+ return;
+ }
+ if (namespace.release() <= 0) {
+ NAMESPACES.remove(metadataProvider);
+ lastUse = namespace;
+ }
}
- if (namespace.release() <= 0) {
- NAMESPACES.remove(metadataProvider);
- namespace.close();
+ if (lastUse != null) {
+ // Closing a manager closes its providers - plugin code again, so not
under the lock.
+ lastUse.close();
}
}
@@ -273,17 +334,20 @@ public class HopVfsNamespaces {
* @return true when a namespace was rebuilt
* @throws HopException if the namespace cannot be rebuilt
*/
- public static synchronized boolean refresh(IVariables variables) throws
HopException {
+ public static boolean refresh(IVariables variables) throws HopException {
HopVfsNamespace namespace = resolve(variables);
if (namespace == null) {
return false;
}
+ // Registers the connections again, so plugin code: the namespace
serializes its own rebuilds.
namespace.rebuild();
return true;
}
- private static synchronized HopVfsNamespace existing(IHopMetadataProvider
provider) {
- return NAMESPACES.get(provider);
+ private static HopVfsNamespace existing(IHopMetadataProvider provider) {
+ synchronized (REGISTRY) {
+ return NAMESPACES.get(provider);
+ }
}
/**
@@ -292,14 +356,20 @@ public class HopVfsNamespaces {
* <p>Leaves the installed scope in place: which runtime this is does not
change because the
* namespaces were closed. A test that installed one puts it back with
{@code setScope(null)}.
*/
- public static synchronized void reset() {
- NAMESPACES.values().forEach(HopVfsNamespace::close);
- NAMESPACES.clear();
+ public static void reset() {
+ List<HopVfsNamespace> open;
+ synchronized (REGISTRY) {
+ open = new ArrayList<>(NAMESPACES.values());
+ NAMESPACES.clear();
+ }
+ open.forEach(HopVfsNamespace::close);
scope.remove();
}
/** How many namespaces are open right now. For tests and diagnostics. */
- public static synchronized int size() {
- return NAMESPACES.size();
+ public static int size() {
+ synchronized (REGISTRY) {
+ return NAMESPACES.size();
+ }
}
}
diff --git
a/core/src/test/java/org/apache/hop/core/vfs/HopVfsNamespaceLockingTest.java
b/core/src/test/java/org/apache/hop/core/vfs/HopVfsNamespaceLockingTest.java
new file mode 100644
index 0000000000..5a6d537d00
--- /dev/null
+++ b/core/src/test/java/org/apache/hop/core/vfs/HopVfsNamespaceLockingTest.java
@@ -0,0 +1,401 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.vfs;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
+import org.apache.commons.vfs2.provider.FileProvider;
+import org.apache.commons.vfs2.provider.ram.RamFileProvider;
+import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.extension.IPluginMock;
+import org.apache.hop.core.plugins.IPlugin;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.vfs.plugin.IVfs;
+import org.apache.hop.core.vfs.plugin.VfsPluginType;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
+import org.apache.hop.metadata.util.HopMetadataInstance;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Registering the named connections of a namespace runs plugin code that
reads metadata, and
+ * reading metadata resolves files through {@link HopVfs}. A Hop Server
running two exported
+ * executions at once deadlocked on exactly that: one thread held the {@link
HopVfs} monitor while
+ * looking its namespace up, the other held the {@link HopVfsNamespaces}
monitor while registering
+ * providers and wanted {@link HopVfs}. Neither of the two properties below
may be given up.
+ */
+class HopVfsNamespaceLockingTest {
+
+ private IPlugin registeredPlugin;
+
+ /** What the test VFS plugin does while its providers are being registered.
*/
+ private final AtomicReference<Runnable> whileRegistering = new
AtomicReference<>(() -> {});
+
+ /** The scheme the test VFS plugin registers for an export's metadata;
"locktest" otherwise. */
+ private final Map<IHopMetadataProvider, String> schemeFor = new
ConcurrentHashMap<>();
+
+ @BeforeAll
+ static void initHop() throws Exception {
+ HopClientEnvironment.init();
+ }
+
+ @BeforeEach
+ void setUp() {
+ HopMetadataInstance.setMetadataProvider(null);
+ HopVfs.setBootstrapVariables(null);
+ HopVfs.reset();
+ registerTestVfsPlugin();
+ // The process wide manager belongs to some other metadata; everything
below needs a
+ // namespace of its own, the way an export on a server does.
+ HopMetadataInstance.setMetadataProvider(mock(MultiMetadataProvider.class));
+ HopVfs.setBootstrapVariables(Variables.getADefaultVariableSpace());
+ HopVfs.getFileSystemManager();
+ }
+
+ @AfterEach
+ void tearDown() {
+ if (registeredPlugin != null) {
+ PluginRegistry.getInstance().removePlugin(VfsPluginType.class,
registeredPlugin);
+ registeredPlugin = null;
+ }
+ HopMetadataInstance.setMetadataProvider(null);
+ HopVfs.setBootstrapVariables(null);
+ HopVfs.reset();
+ }
+
+ @Test
+ @DisplayName("Named connections are registered without holding the
namespaces monitor")
+ void registrationDoesNotHoldTheNamespacesMonitor() {
+ // The Azure plugin, for one, reads its connections with a serializer that
validates its
+ // folder through HopVfs.getFileObject(): plugin code under a global
monitor.
+ List<Boolean> heldWhileRegistering = new CopyOnWriteArrayList<>();
+ whileRegistering.set(() ->
heldWhileRegistering.add(Thread.holdsLock(HopVfsNamespaces.class)));
+
+ IHopMetadataProvider exported = mock(MultiMetadataProvider.class);
+ HopVfsNamespace namespace =
+ HopVfsNamespaces.acquire(variablesOf(exported), exported, "exported
workflow");
+ try {
+ assertNotNull(namespace);
+ assertFalse(
+ heldWhileRegistering.isEmpty(), "the test VFS plugin was not asked
for its providers");
+ assertFalse(
+ heldWhileRegistering.contains(Boolean.TRUE),
+ "HopVfsNamespaces holds its monitor while running VFS plugin code,
which resolves"
+ + " files through HopVfs and so waits for whoever holds the
HopVfs monitor");
+ } finally {
+ HopVfsNamespaces.release(exported);
+ }
+ }
+
+ @Test
+ @DisplayName("Resolving through HopVfs never waits for a namespace being
registered elsewhere")
+ void resolvingDoesNotWaitForAnotherNamespacesRegistration() throws Exception
{
+ // A workflow already running on the server, with its namespace in place.
+ IHopMetadataProvider running = mock(MultiMetadataProvider.class);
+ IVariables runningVariables = variablesOf(running);
+ HopVfsNamespace runningNamespace =
+ HopVfsNamespaces.acquire(runningVariables, running, "running
workflow");
+ assertNotNull(runningNamespace);
+
+ // A second export arrives: its namespace is being registered, and the
plugin is busy reading
+ // metadata - long enough for the first workflow to go and load its
pipeline meanwhile.
+ CountDownLatch registering = new CountDownLatch(1);
+ CountDownLatch letRegistrationFinish = new CountDownLatch(1);
+ whileRegistering.set(
+ () -> {
+ registering.countDown();
+ await(letRegistrationFinish);
+ });
+ IHopMetadataProvider arriving = mock(MultiMetadataProvider.class);
+ Thread secondExport =
+ new Thread(
+ () -> HopVfsNamespaces.acquire(variablesOf(arriving), arriving,
"arriving workflow"),
+ "second export");
+ secondExport.setDaemon(true);
+
+ AtomicReference<DefaultFileSystemManager> resolved = new
AtomicReference<>();
+ Thread loadingItsPipeline =
+ new Thread(
+ () -> resolved.set(HopVfs.getFileSystemManager(runningVariables)),
+ "running workflow loading its pipeline");
+ loadingItsPipeline.setDaemon(true);
+
+ try {
+ secondExport.start();
+ assertTrue(registering.await(10, TimeUnit.SECONDS), "registration never
started");
+
+ loadingItsPipeline.start();
+ loadingItsPipeline.join(TimeUnit.SECONDS.toMillis(5));
+
+ assertFalse(
+ loadingItsPipeline.isAlive(),
+ "HopVfs.getFileSystemManager(variables) is stuck behind the
registration of an"
+ + " unrelated namespace: it takes the HopVfs monitor and then
waits for the"
+ + " HopVfsNamespaces monitor. Had that registration needed
HopVfs, as reading"
+ + " metadata does, both would wait forever.");
+ assertSame(runningNamespace.getFileSystemManager(), resolved.get());
+ } finally {
+ letRegistrationFinish.countDown();
+ secondExport.join(TimeUnit.SECONDS.toMillis(10));
+ loadingItsPipeline.join(TimeUnit.SECONDS.toMillis(10));
+ HopVfsNamespaces.release(arriving);
+ HopVfsNamespaces.release(running);
+ }
+ }
+
+ @Test
+ @DisplayName(
+ "A second user of the same metadata waits for the connections, the
registering"
+ + " thread itself does not")
+ void secondAcquirerWaitsForRegistration() throws Exception {
+ IHopMetadataProvider shared = mock(MultiMetadataProvider.class);
+ IVariables variables = variablesOf(shared);
+ CountDownLatch registering = new CountDownLatch(1);
+ CountDownLatch letRegistrationFinish = new CountDownLatch(1);
+ AtomicReference<HopVfsNamespace> seenWhileRegistering = new
AtomicReference<>();
+ whileRegistering.set(
+ () -> {
+ // Reading metadata resolves files: on this thread that must find
the namespace at
+ // once, connections or not, or registration would wait for itself.
+ seenWhileRegistering.set(HopVfsNamespaces.resolve(variables));
+ registering.countDown();
+ await(letRegistrationFinish);
+ });
+
+ AtomicReference<HopVfsNamespace> first = new AtomicReference<>();
+ Thread workflow =
+ new Thread(
+ () -> first.set(HopVfsNamespaces.acquire(variables, shared,
"workflow")), "workflow");
+ workflow.setDaemon(true);
+ AtomicReference<HopVfsNamespace> second = new AtomicReference<>();
+ Thread itsPipeline =
+ new Thread(
+ () -> second.set(HopVfsNamespaces.acquire(variables, shared, "its
pipeline")),
+ "its pipeline");
+ itsPipeline.setDaemon(true);
+
+ try {
+ workflow.start();
+ assertTrue(registering.await(10, TimeUnit.SECONDS), "registration never
started");
+ itsPipeline.start();
+ itsPipeline.join(500);
+ assertTrue(
+ itsPipeline.isAlive(),
+ "The pipeline got the namespace before its named connections were
registered");
+ assertNotNull(seenWhileRegistering.get(), "re-entrant resolution did not
find the namespace");
+ } finally {
+ letRegistrationFinish.countDown();
+ workflow.join(TimeUnit.SECONDS.toMillis(10));
+ itsPipeline.join(TimeUnit.SECONDS.toMillis(10));
+ }
+ try {
+ assertFalse(workflow.isAlive() || itsPipeline.isAlive());
+ assertNotNull(first.get());
+ assertSame(first.get(), second.get());
+ assertSame(first.get(), seenWhileRegistering.get());
+ assertTrue(
+ first.get().getFileSystemManager().hasProvider("locktest"),
+ "the named connection is not registered");
+ } finally {
+ HopVfsNamespaces.release(shared);
+ HopVfsNamespaces.release(shared);
+ }
+ }
+
+ /**
+ * The whole point of a namespace per export (#8106): its named connections
reach nobody else, and
+ * they go away with it. That has to survive the registration now running
outside the lock.
+ */
+ @Test
+ @DisplayName("Concurrent exports keep their connections apart and close with
their last user")
+ void concurrentExportsKeepTheirProvidersApartAndCloseWithTheLastUser()
throws Exception {
+ int exports = 8;
+ int rounds = 25;
+ List<IHopMetadataProvider> providers = new ArrayList<>();
+ for (int i = 0; i < exports; i++) {
+ IHopMetadataProvider export = mock(MultiMetadataProvider.class);
+ schemeFor.put(export, "export" + i);
+ providers.add(export);
+ }
+ DefaultFileSystemManager processWide = HopVfs.getFileSystemManager();
+ Set<DefaultFileSystemManager> managers = ConcurrentHashMap.newKeySet();
+ List<Throwable> failures = new CopyOnWriteArrayList<>();
+ CountDownLatch go = new CountDownLatch(1);
+ List<Thread> threads = new ArrayList<>();
+
+ for (int i = 0; i < exports; i++) {
+ IHopMetadataProvider export = providers.get(i);
+ String mine = schemeFor.get(export);
+ IVariables variables = variablesOf(export);
+ Runnable oneExecution =
+ () -> {
+ // A workflow and, inside it, a pipeline: two users of the same
namespace.
+ HopVfsNamespace workflow = HopVfsNamespaces.acquire(variables,
export, mine);
+ try {
+ HopVfsNamespace pipeline = HopVfsNamespaces.acquire(variables,
export, mine);
+ try {
+ assertSame(workflow, pipeline);
+ DefaultFileSystemManager manager =
HopVfs.getFileSystemManager(variables);
+ managers.add(manager);
+ assertSame(workflow.getFileSystemManager(), manager);
+ assertTrue(manager.hasProvider(mine), mine + " is missing from
its own namespace");
+ for (String other : schemeFor.values()) {
+ if (!other.equals(mine)) {
+ assertFalse(manager.hasProvider(other), other + " leaked
into " + mine);
+ }
+ }
+ assertFalse(
+ processWide.hasProvider(mine), mine + " leaked into the
process manager");
+ } finally {
+ HopVfsNamespaces.release(export);
+ }
+ } finally {
+ HopVfsNamespaces.release(export);
+ }
+ };
+ threads.add(
+ new Thread(
+ () -> {
+ await(go);
+ for (int round = 0; round < rounds; round++) {
+ try {
+ oneExecution.run();
+ } catch (Throwable t) {
+ failures.add(t);
+ return;
+ }
+ }
+ },
+ "export " + i));
+ }
+ // And bystanders that only resolve, the way a running execution loads its
files.
+ for (int i = 0; i < exports; i++) {
+ IVariables variables = variablesOf(providers.get(i));
+ threads.add(
+ new Thread(
+ () -> {
+ await(go);
+ for (int round = 0; round < rounds * 4; round++) {
+ HopVfs.getFileSystemManager(variables);
+ }
+ },
+ "bystander " + i));
+ }
+ threads.forEach(Thread::start);
+ go.countDown();
+ for (Thread thread : threads) {
+ thread.join(TimeUnit.SECONDS.toMillis(60));
+ assertFalse(thread.isAlive(), thread.getName() + " did not finish");
+ }
+
+ assertTrue(failures.isEmpty(), () -> "failures: " + failures);
+ assertTrue(managers.size() >= exports, "each export must have had a
manager of its own");
+ assertSame(0, HopVfsNamespaces.size(), "namespaces left open after their
last user let go");
+ for (DefaultFileSystemManager manager : managers) {
+ // A closed manager has dropped every provider, the local one included.
+ assertFalse(manager.hasProvider("file"), "a namespace's manager was
never closed");
+ }
+ for (String scheme : schemeFor.values()) {
+ assertFalse(processWide.hasProvider(scheme), scheme + " leaked into the
process manager");
+ }
+ assertTrue(processWide.hasProvider("file"), "the process manager itself
must stay open");
+ }
+
+ private static void await(CountDownLatch latch) {
+ try {
+ if (!latch.await(10, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("Timed out waiting for the test to
move on");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /** The variables of something running against this metadata, the way an
execution's are. */
+ private static Variables variablesOf(IHopMetadataProvider provider) {
+ return new Variables() {
+ @Override
+ public IHopMetadataProvider getMetadataProvider() {
+ return provider;
+ }
+ };
+ }
+
+ /** A VFS plugin that, like the metadata driven ones, does work of its own
while registering. */
+ private void registerTestVfsPlugin() {
+ IVfs vfs =
+ new IVfs() {
+ @Override
+ public String[] getUrlSchemes() {
+ return new String[] {};
+ }
+
+ @Override
+ public FileProvider getProvider() {
+ return new RamFileProvider();
+ }
+
+ @Override
+ public Map<String, FileProvider> getProviders(IVariables variables) {
+ whileRegistering.get().run();
+ Map<String, FileProvider> providers = new HashMap<>();
+ // Each export carries a connection of its own, named after the
export.
+ IHopMetadataProvider owner =
+ variables == null ? null :
variables.findExecutionMetadataProvider();
+ String scheme = owner == null ? null : schemeFor.get(owner);
+ providers.put(scheme == null ? "locktest" : scheme, new
RamFileProvider());
+ return providers;
+ }
+ };
+
+ IPluginMock plugin = mock(IPluginMock.class);
+ when(plugin.getIds()).thenReturn(new String[] {"test-vfs-locking-plugin"});
+ when(plugin.getName()).thenReturn("Test VFS locking plugin");
+ when(plugin.getMainType()).thenReturn((Class) IVfs.class);
+ when(plugin.loadClass(IVfs.class)).thenReturn(vfs);
+ try {
+ PluginRegistry.getInstance().registerPlugin(VfsPluginType.class, plugin);
+ } catch (Exception e) {
+ throw new IllegalStateException("Unable to register the test VFS
plugin", e);
+ }
+ registeredPlugin = plugin;
+ }
+}
diff --git a/integration-tests/hop_server/0019-child-delay.hpl
b/integration-tests/hop_server/0019-child-delay.hpl
new file mode 100644
index 0000000000..d8894a3a31
--- /dev/null
+++ b/integration-tests/hop_server/0019-child-delay.hpl
@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters/>
+ <name>0019-child-delay</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Small worker pipeline that occupies a server slot for 2
seconds</description>
+ <created_user>-</created_user>
+ <modified_user>-</modified_user>
+ <created_date>2026/08/18 12:53:30.940</created_date>
+ <modified_date>2026/08/18 12:53:30.940</modified_date>
+ </info>
+ <transform>
+ <type>RowGenerator</type>
+ <name>1 row</name>
+ <never_ending>N</never_ending>
+ <interval_in_ms>5000</interval_in_ms>
+ <row_time_field/>
+ <last_time_field/>
+ <limit>1</limit>
+ <fields/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>Delay</type>
+ <name>Delay 1s</name>
+ <timeout>1</timeout>
+ <scaletime>seconds</scaletime>
+ <scaletime_from_field>N</scaletime_from_field>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>272</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>Dummy</type>
+ <name>Dummy</name>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>448</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <order>
+ <hop>
+ <from>1 row</from>
+ <to>Delay 1s</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Delay 1s</from>
+ <to>Dummy</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <notepads/>
+ <attributes/>
+ <transform_error_handling/>
+</pipeline>
diff --git a/integration-tests/hop_server/0019-child-delay.hwf
b/integration-tests/hop_server/0019-child-delay.hwf
new file mode 100644
index 0000000000..e2d90998d2
--- /dev/null
+++ b/integration-tests/hop_server/0019-child-delay.hwf
@@ -0,0 +1,99 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>0019-child-delay</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Small worker workflow that occupies a server slot for 2
seconds</description>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>64</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0019-child-delay.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0019-child-delay.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <set_logfile>N</set_logfile>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <wait_until_finished>Y</wait_until_finished>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <set_append_logfile>N</set_append_logfile>
+ <parallel>N</parallel>
+ <xloc>240</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Success</name>
+ <description/>
+ <type>SUCCESS</type>
+ <attributes/>
+ <parallel>N</parallel>
+ <xloc>448</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0019-child-delay.hpl</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>0019-child-delay.hpl</from>
+ <to>Success</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/hop_server/0019-drive-200-workflows.hpl
b/integration-tests/hop_server/0019-drive-200-workflows.hpl
new file mode 100644
index 0000000000..103415bb97
--- /dev/null
+++ b/integration-tests/hop_server/0019-drive-200-workflows.hpl
@@ -0,0 +1,181 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0019-drive-200-workflows</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Submit 200 child workflows as exports to the one Hop Server,
eight at a time</description>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>200 rows</from>
+ <to>Workflow executor</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Workflow executor</from>
+ <to>Count successes</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Count successes</from>
+ <to>Set EXEC_OK</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>200 rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ </fields>
+ <limit>200</limit>
+ <never_ending>N</never_ending>
+ <interval_in_ms>5000</interval_in_ms>
+ <row_time_field/>
+ <last_time_field/>
+ <attributes/>
+ <GUI>
+ <xloc>80</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Workflow executor</name>
+ <type>WorkflowExecutor</type>
+ <description/>
+ <distribute>Y</distribute>
+ <copies>8</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <run_configuration>remote</run_configuration>
+ <filename>${PROJECT_HOME}/0019-child-delay.hwf</filename>
+ <group_size>1</group_size>
+ <group_field/>
+ <group_time/>
+ <parameters>
+ <inherit_all_vars>Y</inherit_all_vars>
+ </parameters>
+ <execution_result_target_transform>Count
successes</execution_result_target_transform>
+ <execution_time_field>ExecutionTime</execution_time_field>
+ <execution_result_field>ExecutionResult</execution_result_field>
+ <execution_errors_field>ExecutionNrErrors</execution_errors_field>
+ <execution_lines_read_field/>
+ <execution_lines_written_field/>
+ <execution_lines_input_field/>
+ <execution_lines_output_field/>
+ <execution_lines_rejected_field/>
+ <execution_lines_updated_field/>
+ <execution_lines_deleted_field/>
+ <execution_files_retrieved_field/>
+ <execution_exit_status_field/>
+ <execution_log_text_field/>
+ <execution_log_channelid_field/>
+ <result_rows_target_transform/>
+ <result_files_target_transform/>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Count successes</name>
+ <type>GroupBy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <add_linenr>N</add_linenr>
+ <all_rows>N</all_rows>
+ <directory>${java.io.tmpdir}</directory>
+ <fields>
+ <field>
+ <aggregate>okCount</aggregate>
+ <type>COUNT_ANY</type>
+ </field>
+ <field>
+ <aggregate>errorSum</aggregate>
+ <subject>ExecutionNrErrors</subject>
+ <type>SUM</type>
+ </field>
+ </fields>
+ <give_back_row>N</give_back_row>
+ <group>
+ </group>
+ <ignore_aggregate>N</ignore_aggregate>
+ <prefix>grp</prefix>
+ <attributes/>
+ <GUI>
+ <xloc>512</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Set EXEC_OK</name>
+ <type>SetVariable</type>
+ <description/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <field_name>okCount</field_name>
+ <variable_name>EXEC_OK</variable_name>
+ <variable_type>PARENT_WORKFLOW</variable_type>
+ <default_value>0</default_value>
+ </field>
+ <field>
+ <field_name>errorSum</field_name>
+ <variable_name>EXEC_ERRORS</variable_name>
+ <variable_type>PARENT_WORKFLOW</variable_type>
+ <default_value>0</default_value>
+ </field>
+ </fields>
+ <use_formatting>Y</use_formatting>
+ <attributes/>
+ <GUI>
+ <xloc>704</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/hop_server/main-0019-concurrent-exports-single-server.hwf
b/integration-tests/hop_server/main-0019-concurrent-exports-single-server.hwf
new file mode 100644
index 0000000000..bbc8e3a069
--- /dev/null
+++
b/integration-tests/hop_server/main-0019-concurrent-exports-single-server.hwf
@@ -0,0 +1,168 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0019-concurrent-exports-single-server</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Fire 200 exported child workflows at the one Hop Server, eight
at a time. Two exports starting together used to deadlock the server between
HopVfs and HopVfsNamespaces; every one of them must finish.</description>
+ <created_user>-</created_user>
+ <modified_user>-</modified_user>
+ <created_date>2026/09/18 17:30:00.000</created_date>
+ <modified_date>2026/09/18 17:30:00.000</modified_date>
+ <parameters/>
+ <actions>
+ <action>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <DayOfMonth>1</DayOfMonth>
+ <weekDay>1</weekDay>
+ <minutes>0</minutes>
+ <hour>12</hour>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <xloc>48</xloc>
+ <yloc>160</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <replacevars>Y</replacevars>
+ <fields>
+ <field>
+ <variable_name>EXEC_OK</variable_name>
+ <variable_value>0</variable_value>
+ <variable_type>CURRENT_WORKFLOW</variable_type>
+ </field>
+ <field>
+ <variable_name>EXEC_ERRORS</variable_name>
+ <variable_value>0</variable_value>
+ <variable_type>CURRENT_WORKFLOW</variable_type>
+ </field>
+ </fields>
+ <file_variable_type>CURRENT_WORKFLOW</file_variable_type>
+ <name>Init variables</name>
+ <description/>
+ <type>SET_VARIABLES</type>
+ <attributes/>
+ <xloc>192</xloc>
+ <yloc>160</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <filename>${PROJECT_HOME}/0019-drive-200-workflows.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <create_parent_folder>N</create_parent_folder>
+ <set_logfile>N</set_logfile>
+ <set_append_logfile>N</set_append_logfile>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <wait_until_finished>Y</wait_until_finished>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <run_configuration>local</run_configuration>
+ <name>Drive 200 workflows</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <xloc>384</xloc>
+ <yloc>160</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <valuetype>variable</valuetype>
+ <fieldtype>number</fieldtype>
+ <variablename>EXEC_OK</variablename>
+ <comparevalue>200</comparevalue>
+ <successcondition>equal</successcondition>
+ <successwhenvarset>N</successwhenvarset>
+ <successbooleancondition>true</successbooleancondition>
+ <successnumbercondition>equal</successnumbercondition>
+ <name>200 executions</name>
+ <description/>
+ <type>SIMPLE_EVAL</type>
+ <attributes/>
+ <xloc>576</xloc>
+ <yloc>272</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <valuetype>variable</valuetype>
+ <fieldtype>number</fieldtype>
+ <variablename>EXEC_ERRORS</variablename>
+ <comparevalue>0</comparevalue>
+ <successcondition>equal</successcondition>
+ <successwhenvarset>N</successwhenvarset>
+ <successbooleancondition>true</successbooleancondition>
+ <successnumbercondition>equal</successnumbercondition>
+ <name>No execution errors</name>
+ <description/>
+ <type>SIMPLE_EVAL</type>
+ <attributes/>
+ <xloc>768</xloc>
+ <yloc>272</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Init variables</to>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Init variables</from>
+ <to>Drive 200 workflows</to>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Drive 200 workflows</from>
+ <to>200 executions</to>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>200 executions</from>
+ <to>No execution errors</to>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ </hops>
+ <notepads/>
+ <attributes/>
+</workflow>
