This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new b8b863bb4a Export pipelines/workflows referenced from metadata
objects, fixes #3368 (#7541)
b8b863bb4a is described below
commit b8b863bb4ae27503aaa10f934df2cebc2ad80127
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Jul 16 14:46:02 2026 +0200
Export pipelines/workflows referenced from metadata objects, fixes #3368
(#7541)
---
.../org/apache/hop/core/variables/IVariables.java | 14 +
.../org/apache/hop/core/variables/Variables.java | 30 +-
.../java/org/apache/hop/resource/ResourceUtil.java | 227 ++++++++++++++-
.../hop_server/0008-trigger-pipeline.hpl | 85 ++++++
.../0008-verify-metadata-referenced-pipeline.hwf | 134 +++++++++
.../hop_server/0009-trigger-pipeline.hpl | 76 ++++++
.../hop_server/0009-verify-resolver-pipeline.hwf | 133 +++++++++
...main-0008-test-metadata-referenced-pipeline.hwf | 95 +++++++
.../main-0009-test-variable-resolver-pipeline.hwf | 98 +++++++
.../metadata/pipeline-log/0008-metadata-log.json | 17 ++
.../metadata/variable-resolver/resolver-0009.json | 13 +
.../hop_server/reflection/0008-log-marker.hpl | 151 ++++++++++
.../hop_server/reflection/0009-resolver.hpl | 174 ++++++++++++
.../export/MetadataResourceExportTest.java | 303 +++++++++++++++++++++
.../kafka/consumer/KafkaConsumerInputMeta.java | 12 +
.../kafka/consumer/KafkaConsumerInputMetaTest.java | 18 ++
16 files changed, 1574 insertions(+), 6 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/variables/IVariables.java
b/core/src/main/java/org/apache/hop/core/variables/IVariables.java
index 969637c06e..ebe9c5ab5c 100644
--- a/core/src/main/java/org/apache/hop/core/variables/IVariables.java
+++ b/core/src/main/java/org/apache/hop/core/variables/IVariables.java
@@ -20,6 +20,7 @@ package org.apache.hop.core.variables;
import java.util.Map;
import org.apache.hop.core.exception.HopValueException;
import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
/** Interface to implement variable sensitive objects. */
public interface IVariables {
@@ -146,4 +147,17 @@ public interface IVariables {
* @throws HopValueException In case there is a String conversion error
*/
String resolve(String aString, IRowMeta rowMeta, Object[] rowData) throws
HopValueException;
+
+ /**
+ * The metadata provider associated with the execution this variable space
belongs to (e.g. a
+ * running pipeline or workflow). It is used, for example, to look up
variable resolvers so that
+ * they are resolved against the metadata of the current execution - the
exported/bundled metadata
+ * on a remote server - rather than only the process-global metadata.
Returns {@code null} by
+ * default; execution engines expose their own provider.
+ *
+ * @return the metadata provider of this execution, or {@code null} if none.
+ */
+ default IHopMetadataProvider getMetadataProvider() {
+ return null;
+ }
}
diff --git a/core/src/main/java/org/apache/hop/core/variables/Variables.java
b/core/src/main/java/org/apache/hop/core/variables/Variables.java
index b04d37b3ce..cfda4fdfdb 100644
--- a/core/src/main/java/org/apache/hop/core/variables/Variables.java
+++ b/core/src/main/java/org/apache/hop/core/variables/Variables.java
@@ -35,8 +35,8 @@ import org.apache.hop.core.row.value.ValueMetaBase;
import org.apache.hop.core.util.StringUtil;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.resolver.VariableResolver;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.metadata.api.IHopMetadataSerializer;
-import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
import org.apache.hop.metadata.util.HopMetadataInstance;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
@@ -171,6 +171,26 @@ public class Variables implements IVariables {
return resolved;
}
+ /**
+ * Walk this variable space and its parents to find the metadata provider of
the current execution
+ * (an execution engine exposes its own provider through {@link
+ * IVariables#getMetadataProvider()}).
+ *
+ * @return the first non-null metadata provider found, or {@code null} if
none.
+ */
+ private IHopMetadataProvider findExecutionMetadataProvider() {
+ IVariables space = this;
+ int guard = 0;
+ while (space != null && guard++ < 100) {
+ IHopMetadataProvider provider = space.getMetadataProvider();
+ if (provider != null) {
+ return provider;
+ }
+ space = space.getParentVariables();
+ }
+ return null;
+ }
+
private String substituteVariableResolvers(String input) {
String resolved = input;
int startIndex = 0;
@@ -212,7 +232,13 @@ public class Variables implements IVariables {
}
try {
- MultiMetadataProvider provider =
HopMetadataInstance.getMetadataProvider();
+ // Resolve against the metadata of the current execution (e.g. the
exported/bundled
+ // metadata on a remote server) when available, otherwise the
process-global metadata.
+ //
+ IHopMetadataProvider provider = findExecutionMetadataProvider();
+ if (provider == null) {
+ provider = HopMetadataInstance.getMetadataProvider();
+ }
IHopMetadataSerializer<VariableResolver> serializer =
provider.getSerializer(VariableResolver.class);
diff --git a/engine/src/main/java/org/apache/hop/resource/ResourceUtil.java
b/engine/src/main/java/org/apache/hop/resource/ResourceUtil.java
index 4ebcf36bae..f5b7770ef3 100644
--- a/engine/src/main/java/org/apache/hop/resource/ResourceUtil.java
+++ b/engine/src/main/java/org/apache/hop/resource/ResourceUtil.java
@@ -18,9 +18,14 @@
package org.apache.hop.resource;
import java.io.IOException;
+import java.lang.reflect.Field;
import java.nio.charset.StandardCharsets;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.commons.lang3.StringUtils;
@@ -34,8 +39,14 @@ import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.core.xml.IXml;
import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+import org.apache.hop.metadata.api.IHopMetadata;
import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.metadata.util.ReflectionUtil;
import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.workflow.WorkflowMeta;
public class ResourceUtil {
@@ -89,6 +100,18 @@ public class ResourceUtil {
// See if we need to rename folders and if we need to expose those as
variables in the
// execution configuration...
//
+ // Bundle the pipelines/workflows that are referenced from metadata
objects (Pipeline Log,
+ // Workflow Log, Pipeline Probe, variable resolvers, web services,
...) and rewrite those
+ // references to point at the copies inside the ZIP archive. Otherwise
those pipelines are
+ // missing on the remote server (see #3368).
+ //
+ // This is done first because bundling a referenced pipeline can add
new named-resource
+ // folders (DATA_PATH_n) which then need a value assigned below.
+ //
+ SerializableMetadataProvider exportMetadataProvider =
+ exportMetadataReferencedResources(
+ variables, definitions, namingInterface, metadataProvider);
+
// See if we need to rename named resource folders...
// We have a list of these in the naming interface
//
@@ -146,10 +169,7 @@ public class ResourceUtil {
ZipEntry jsonEntry = new ZipEntry("metadata.json");
jsonEntry.setComment("Export of the client metadata");
out.putNextEntry(jsonEntry);
- out.write(
- new SerializableMetadataProvider(metadataProvider)
- .toJson()
- .getBytes(StandardCharsets.UTF_8));
+
out.write(exportMetadataProvider.toJson().getBytes(StandardCharsets.UTF_8));
out.closeEntry();
String zipURL = fileObject.getName().toString();
@@ -243,6 +263,205 @@ public class ResourceUtil {
}
}
+ /**
+ * Bundle the pipelines and workflows that are referenced from metadata
objects (e.g. Pipeline
+ * Log, Workflow Log, Pipeline Probe, variable resolvers, web services) into
the export {@code
+ * definitions} and rewrite those references so they resolve inside the ZIP
archive on the remote
+ * server. Metadata objects are only serialized to {@code metadata.json};
unlike transforms and
+ * actions they are never walked by the normal resource export, so their
referenced pipelines went
+ * missing on the server (#3368).
+ *
+ * <p>The rewriting is done on an independent deep copy of the metadata so
the caller's live
+ * metadata objects are never modified.
+ *
+ * @return a metadata provider holding the rewritten copy, to be serialized
into the ZIP archive.
+ */
+ static SerializableMetadataProvider exportMetadataReferencedResources(
+ IVariables variables,
+ Map<String, ResourceDefinition> definitions,
+ IResourceNaming namingInterface,
+ IHopMetadataProvider metadataProvider)
+ throws HopException {
+
+ // Deep copy through a JSON round-trip so we never mutate the caller's
live metadata objects
+ // (for in-memory providers the objects would otherwise be shared by
reference).
+ //
+ SerializableMetadataProvider exportMetadataProvider =
+ new SerializableMetadataProvider(
+ new SerializableMetadataProvider(metadataProvider).toJson());
+
+ Set<Object> visited = Collections.newSetFromMap(new IdentityHashMap<>());
+
+ for (Class<IHopMetadata> metadataClass :
exportMetadataProvider.getMetadataClasses()) {
+ IHopMetadataSerializer<IHopMetadata> serializer =
+ exportMetadataProvider.getSerializer(metadataClass);
+ for (String name : serializer.listObjectNames()) {
+ IHopMetadata object = serializer.load(name);
+ if (rewriteMetadataFileReferences(
+ object, visited, variables, definitions, namingInterface,
metadataProvider)) {
+ serializer.save(object);
+ }
+ }
+ }
+ return exportMetadataProvider;
+ }
+
+ /**
+ * Recursively walk a metadata object, bundling and rewriting every {@link
+ * HopMetadataPropertyType#PIPELINE_FILE} / {@link
HopMetadataPropertyType#WORKFLOW_FILE} String
+ * reference it (or a nested metadata object / list) holds.
+ *
+ * @return true if anything was rewritten.
+ */
+ private static boolean rewriteMetadataFileReferences(
+ Object object,
+ Set<Object> visited,
+ IVariables variables,
+ Map<String, ResourceDefinition> definitions,
+ IResourceNaming namingInterface,
+ IHopMetadataProvider metadataProvider)
+ throws HopException {
+
+ if (object == null || !visited.add(object)) {
+ return false;
+ }
+ boolean changed = false;
+
+ for (Field field : ReflectionUtil.findAllFields(object.getClass())) {
+ HopMetadataProperty property =
field.getAnnotation(HopMetadataProperty.class);
+ if (property == null) {
+ continue;
+ }
+ HopMetadataPropertyType type = property.hopMetadataPropertyType();
+ boolean fileReference =
+ type == HopMetadataPropertyType.PIPELINE_FILE
+ || type == HopMetadataPropertyType.WORKFLOW_FILE
+ || type == HopMetadataPropertyType.HOP_FILE;
+
+ Object value = getMetadataFieldValue(object, field);
+ if (value == null) {
+ continue;
+ }
+
+ if (value instanceof String stringValue) {
+ if (fileReference && StringUtils.isNotEmpty(stringValue)) {
+ String rewritten =
+ exportReferencedFile(
+ stringValue, type, variables, definitions, namingInterface,
metadataProvider);
+ if (rewritten != null && !rewritten.equals(stringValue)) {
+ changed |= setMetadataFieldValue(object, field, rewritten);
+ }
+ }
+ } else if (value instanceof List<?> list) {
+ for (int i = 0; i < list.size(); i++) {
+ Object item = list.get(i);
+ if (item instanceof String stringItem) {
+ if (fileReference && StringUtils.isNotEmpty(stringItem)) {
+ String rewritten =
+ exportReferencedFile(
+ stringItem, type, variables, definitions,
namingInterface, metadataProvider);
+ if (rewritten != null && !rewritten.equals(stringItem)) {
+ @SuppressWarnings("unchecked")
+ List<Object> objectList = (List<Object>) list;
+ objectList.set(i, rewritten);
+ changed = true;
+ }
+ }
+ } else if (isRecursableMetadataObject(item)) {
+ changed |=
+ rewriteMetadataFileReferences(
+ item, visited, variables, definitions, namingInterface,
metadataProvider);
+ }
+ }
+ } else if (isRecursableMetadataObject(value)) {
+ changed |=
+ rewriteMetadataFileReferences(
+ value, visited, variables, definitions, namingInterface,
metadataProvider);
+ }
+ }
+ return changed;
+ }
+
+ private static Object getMetadataFieldValue(Object object, Field field) {
+ try {
+ return ReflectionUtil.getFieldValue(
+ object, field.getName(), field.getType() == boolean.class);
+ } catch (Exception e) {
+ // A metadata property without a conventional getter must never break
the whole export;
+ // just skip that field (its reference, if any, is left unchanged).
+ return null;
+ }
+ }
+
+ private static boolean setMetadataFieldValue(Object object, Field field,
String value) {
+ try {
+ ReflectionUtil.setFieldValue(object, field.getName(), String.class,
value);
+ return true;
+ } catch (Exception e) {
+ // No usable setter: leave the reference unchanged rather than break the
export.
+ return false;
+ }
+ }
+
+ /** Only recurse into Hop metadata objects, not JDK types, enums or
primitives. */
+ private static boolean isRecursableMetadataObject(Object value) {
+ if (value == null) {
+ return false;
+ }
+ Class<?> clazz = value.getClass();
+ return !clazz.isEnum() && !clazz.isPrimitive() &&
clazz.getName().startsWith("org.apache.hop.");
+ }
+
+ /**
+ * Load the referenced pipeline or workflow, export its resources into
{@code definitions} and
+ * return a reference that resolves against the ZIP archive on the server
({@code
+ * ${Internal.Entry.Current.Folder}/<bundled name>}). For an ambiguous {@link
+ * HopMetadataPropertyType#HOP_FILE} reference the type is decided from the
{@code .hwf}/{@code
+ * .hpl} extension (defaulting to a pipeline). A reference that cannot be
resolved to an existing
+ * file, or that fails to load/export, is left unchanged: the export must
not fail on a peripheral
+ * or stale metadata reference (the server logs and skips it).
+ */
+ private static String exportReferencedFile(
+ String filename,
+ HopMetadataPropertyType type,
+ IVariables variables,
+ Map<String, ResourceDefinition> definitions,
+ IResourceNaming namingInterface,
+ IHopMetadataProvider metadataProvider) {
+
+ String realFilename = variables.resolve(filename);
+ try {
+ if (!HopVfs.getFileObject(realFilename).exists()) {
+ return filename;
+ }
+ } catch (Exception e) {
+ // Can't tell whether it exists; leave the reference untouched rather
than break the export.
+ return filename;
+ }
+
+ boolean workflowFile =
+ type == HopMetadataPropertyType.WORKFLOW_FILE
+ || (type == HopMetadataPropertyType.HOP_FILE
+ && realFilename.toLowerCase().endsWith(".hwf"));
+
+ try {
+ String bundledName;
+ if (workflowFile) {
+ WorkflowMeta workflowMeta = new WorkflowMeta(variables, realFilename,
metadataProvider);
+ bundledName =
+ workflowMeta.exportResources(variables, definitions,
namingInterface, metadataProvider);
+ } else {
+ PipelineMeta pipelineMeta = new PipelineMeta(realFilename,
metadataProvider, variables);
+ bundledName =
+ pipelineMeta.exportResources(variables, definitions,
namingInterface, metadataProvider);
+ }
+ return "${" + Const.INTERNAL_VARIABLE_ENTRY_CURRENT_FOLDER + "}/" +
bundledName;
+ } catch (Exception e) {
+ // A peripheral metadata reference must not break the whole export;
leave it unchanged.
+ return filename;
+ }
+ }
+
public static String getExplanation(
String zipFilename, String launchFile, IResourceExport
resourceExportInterface) {
diff --git a/integration-tests/hop_server/0008-trigger-pipeline.hpl
b/integration-tests/hop_server/0008-trigger-pipeline.hpl
new file mode 100644
index 0000000000..64ff6a7fa8
--- /dev/null
+++ b/integration-tests/hop_server/0008-trigger-pipeline.hpl
@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0008-trigger-pipeline</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>A trivial pipeline. Its execution triggers the Pipeline Log
metadata object scoped to
+ it, which runs the referenced logging pipeline. Used to prove that a
metadata-referenced
+ pipeline is exported to the remote server (#3368).</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2024/01/01 00:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2024/01/01 00:00:00.000</modified_date>
+ <key_for_session_key/>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ </order>
+ <transform>
+ <name>Generate rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>trigger</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif>go</nullif>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <limit>1</limit>
+ <never_ending>N</never_ending>
+ <interval_in_ms>5000</interval_in_ms>
+ <row_time_field>now</row_time_field>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <attributes/>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/hop_server/0008-verify-metadata-referenced-pipeline.hwf
b/integration-tests/hop_server/0008-verify-metadata-referenced-pipeline.hwf
new file mode 100644
index 0000000000..0d4c2d0235
--- /dev/null
+++ b/integration-tests/hop_server/0008-verify-metadata-referenced-pipeline.hwf
@@ -0,0 +1,134 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>0008-verify-metadata-referenced-pipeline</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Runs entirely on the remote Hop Server (shipped there by the
"remote" run configuration
+ of the parent workflow). It runs a trigger pipeline whose start fires a
Pipeline Log metadata
+ object; that log object references a separate logging pipeline which
writes a marker file. The
+ check below - on the same server - confirms the marker exists, i.e. the
metadata-referenced
+ logging pipeline was actually bundled into the export and executed
(#3368). Without the fix the
+ logging pipeline is not exported, the server logs "... pipeline file
couldn't be found", the
+ marker is never written and this check fails.</description>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2024/01/01 00:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2024/01/01 00:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>50</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Cleanup marker</name>
+ <description>Remove any marker left by a previous run so the check can
only succeed on a marker
+ written now.</description>
+ <type>DELETE_FILE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/output-0008/marker.txt</filename>
+ <fail_if_file_not_exists>N</fail_if_file_not_exists>
+ <parallel>N</parallel>
+ <xloc>150</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run trigger pipeline</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0008-trigger-pipeline.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <run_configuration>local</run_configuration>
+ <wait_until_finished>Y</wait_until_finished>
+ <create_parent_folder>N</create_parent_folder>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <set_append_logfile>N</set_append_logfile>
+ <parallel>N</parallel>
+ <xloc>300</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Verify logging pipeline ran</name>
+ <description/>
+ <type>FILE_EXISTS</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/output-0008/marker.txt</filename>
+ <parallel>N</parallel>
+ <xloc>470</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Cleanup marker</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>Cleanup marker</from>
+ <to>Run trigger pipeline</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>Run trigger pipeline</from>
+ <to>Verify logging pipeline ran</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/hop_server/0009-trigger-pipeline.hpl
b/integration-tests/hop_server/0009-trigger-pipeline.hpl
new file mode 100644
index 0000000000..df496729df
--- /dev/null
+++ b/integration-tests/hop_server/0009-trigger-pipeline.hpl
@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0009-trigger-pipeline</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Resolves a variable through the resolver-0009 Variable
Resolver
+ (#{resolver-0009:...}). Resolving it runs the resolver's referenced
pipeline, which is what
+ this test proves gets exported to the remote server
(#3368).</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2024/01/01 00:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2024/01/01 00:00:00.000</modified_date>
+ <key_for_session_key/>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ </order>
+ <transform>
+ <name>Resolve variable</name>
+ <type>GetVariable</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <length>-1</length>
+ <name>resolved</name>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <variable>#{resolver-0009:trigger:resolved}</variable>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/hop_server/0009-verify-resolver-pipeline.hwf
b/integration-tests/hop_server/0009-verify-resolver-pipeline.hwf
new file mode 100644
index 0000000000..ff9ab3db00
--- /dev/null
+++ b/integration-tests/hop_server/0009-verify-resolver-pipeline.hwf
@@ -0,0 +1,133 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>0009-verify-resolver-pipeline</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Runs entirely on the remote Hop Server (shipped there by the
"remote" run configuration
+ of the parent workflow). It runs a trigger pipeline that resolves
#{resolver-0009:...}; resolving
+ it runs the Variable Resolver's referenced pipeline, which writes a marker
file. The check below
+ - on the same server - confirms the marker exists, i.e. the
metadata-referenced resolver pipeline
+ was bundled into the export and executed (#3368), and the resolver was
resolved against the
+ exported metadata rather than the server's own.</description>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2024/01/01 00:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2024/01/01 00:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>50</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Cleanup marker</name>
+ <description>Remove any marker left by a previous run so the check can
only succeed on a marker
+ written now.</description>
+ <type>DELETE_FILE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/output-0009/resolver-marker.txt</filename>
+ <fail_if_file_not_exists>N</fail_if_file_not_exists>
+ <parallel>N</parallel>
+ <xloc>150</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run trigger pipeline</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0009-trigger-pipeline.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <run_configuration>local</run_configuration>
+ <wait_until_finished>Y</wait_until_finished>
+ <create_parent_folder>N</create_parent_folder>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <set_append_logfile>N</set_append_logfile>
+ <parallel>N</parallel>
+ <xloc>300</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Verify resolver pipeline ran</name>
+ <description/>
+ <type>FILE_EXISTS</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/output-0009/resolver-marker.txt</filename>
+ <parallel>N</parallel>
+ <xloc>470</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Cleanup marker</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>Cleanup marker</from>
+ <to>Run trigger pipeline</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>Run trigger pipeline</from>
+ <to>Verify resolver pipeline ran</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/hop_server/main-0008-test-metadata-referenced-pipeline.hwf
b/integration-tests/hop_server/main-0008-test-metadata-referenced-pipeline.hwf
new file mode 100644
index 0000000000..dcb587ba23
--- /dev/null
+++
b/integration-tests/hop_server/main-0008-test-metadata-referenced-pipeline.hwf
@@ -0,0 +1,95 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0008-test-metadata-referenced-pipeline</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Regression test for issue #3368: a pipeline referenced from a
metadata object (here a
+ Pipeline Log) must be exported to a remote Hop Server. The inner workflow
is executed remotely
+ with the "remote" run configuration (export_resources=true). On the server
a trigger pipeline
+ fires the Pipeline Log, which runs its referenced logging pipeline and
writes a marker; the
+ inner workflow then checks the marker. Before the fix the logging pipeline
was not bundled into
+ the export, so it "couldn't be found", the marker was never written and
the workflow failed.</description>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2024/01/01 00:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2024/01/01 00:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>50</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Verify metadata-referenced pipeline on server</name>
+ <description/>
+ <type>WORKFLOW</type>
+ <attributes/>
+ <run_configuration>remote</run_configuration>
+
<filename>${PROJECT_HOME}/0008-verify-metadata-referenced-pipeline.hwf</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <wait_until_finished>Y</wait_until_finished>
+ <create_parent_folder>N</create_parent_folder>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <set_append_logfile>N</set_append_logfile>
+ <parallel>N</parallel>
+ <xloc>224</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Verify metadata-referenced pipeline on server</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/hop_server/main-0009-test-variable-resolver-pipeline.hwf
b/integration-tests/hop_server/main-0009-test-variable-resolver-pipeline.hwf
new file mode 100644
index 0000000000..43f4c7a4a8
--- /dev/null
+++ b/integration-tests/hop_server/main-0009-test-variable-resolver-pipeline.hwf
@@ -0,0 +1,98 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0009-test-variable-resolver-pipeline</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Regression test for issue #3368 (and the
variable-resolver/exported-metadata fix): a
+ pipeline referenced from a metadata object through a nested object (a
Variable Resolver wrapping
+ a pipeline resolver) must be exported to a remote Hop Server AND the
resolver must be resolved
+ against the exported metadata. The inner workflow is executed remotely
with the "remote" run
+ configuration (export_resources=true). On the server a trigger pipeline
resolves
+ #{resolver-0009:...}, which runs the resolver's referenced pipeline and
writes a marker; the
+ inner workflow then checks the marker. Before the fix the resolver
pipeline was not bundled (and
+ the resolver was resolved against the server's own metadata), so
resolution failed, the marker
+ was never written and the workflow failed.</description>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2024/01/01 00:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2024/01/01 00:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>50</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Verify resolver pipeline on server</name>
+ <description/>
+ <type>WORKFLOW</type>
+ <attributes/>
+ <run_configuration>remote</run_configuration>
+ <filename>${PROJECT_HOME}/0009-verify-resolver-pipeline.hwf</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <wait_until_finished>Y</wait_until_finished>
+ <create_parent_folder>N</create_parent_folder>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <set_append_logfile>N</set_append_logfile>
+ <parallel>N</parallel>
+ <xloc>224</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Verify resolver pipeline on server</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/hop_server/metadata/pipeline-log/0008-metadata-log.json
b/integration-tests/hop_server/metadata/pipeline-log/0008-metadata-log.json
new file mode 100644
index 0000000000..0ff6ce736c
--- /dev/null
+++ b/integration-tests/hop_server/metadata/pipeline-log/0008-metadata-log.json
@@ -0,0 +1,17 @@
+{
+ "name": "0008-metadata-log",
+ "enabled": true,
+ "loggingParentsOnly": false,
+ "failParentOnLoggingFailure": false,
+ "pipelineFilename": "${PROJECT_HOME}/reflection/0008-log-marker.hpl",
+ "executingAtStart": true,
+ "executingPeriodically": false,
+ "intervalInSeconds": "30",
+ "executingAtEnd": false,
+ "logLevel": "BASIC",
+ "pipelinesToLog": [
+ {
+ "pipelineToLogFilename": "${PROJECT_HOME}/0008-trigger-pipeline.hpl"
+ }
+ ]
+}
diff --git
a/integration-tests/hop_server/metadata/variable-resolver/resolver-0009.json
b/integration-tests/hop_server/metadata/variable-resolver/resolver-0009.json
new file mode 100644
index 0000000000..7c45142af4
--- /dev/null
+++ b/integration-tests/hop_server/metadata/variable-resolver/resolver-0009.json
@@ -0,0 +1,13 @@
+{
+ "virtualPath": "",
+ "name": "resolver-0009",
+ "description": "Resolves a variable by running a referenced pipeline. Used
to verify that a pipeline referenced from a Variable Resolver metadata object
is exported to the remote server (#3368).",
+ "variable-resolver": {
+ "Variable-Resolver-Pipeline": {
+ "outputTransformName": "OUTPUT",
+ "filename": "${PROJECT_HOME}/reflection/0009-resolver.hpl",
+ "expressionVariableName": "VARIABLE_TO_RESOLVE",
+ "runConfigurationName": "local"
+ }
+ }
+}
diff --git a/integration-tests/hop_server/reflection/0008-log-marker.hpl
b/integration-tests/hop_server/reflection/0008-log-marker.hpl
new file mode 100644
index 0000000000..b219fff0cd
--- /dev/null
+++ b/integration-tests/hop_server/reflection/0008-log-marker.hpl
@@ -0,0 +1,151 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0008-log-marker</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Logging pipeline referenced by a Pipeline Log metadata
object. When the referenced
+ pipeline is correctly bundled into the resource export (#3368) this runs
on the remote server
+ and writes a marker file, which the test then verifies.</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2024/01/01 00:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2024/01/01 00:00:00.000</modified_date>
+ <key_for_session_key/>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Generate rows</from>
+ <to>Write marker</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Generate rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>marker</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif>metadata-referenced-pipeline-executed</nullif>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <limit>1</limit>
+ <never_ending>N</never_ending>
+ <interval_in_ms>5000</interval_in_ms>
+ <row_time_field>now</row_time_field>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <attributes/>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Write marker</name>
+ <type>TextFileOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <separator>;</separator>
+ <enclosure>"</enclosure>
+ <enclosure_forced>N</enclosure_forced>
+ <enclosure_fix_disabled>N</enclosure_fix_disabled>
+ <header>Y</header>
+ <footer>N</footer>
+ <format>DOS</format>
+ <compression>None</compression>
+ <encoding>UTF-8</encoding>
+ <endedLine/>
+ <fileNameInField>N</fileNameInField>
+ <fileNameField/>
+ <create_parent_folder>Y</create_parent_folder>
+ <file>
+ <name>${PROJECT_HOME}/output-0008/marker</name>
+ <servlet_output>N</servlet_output>
+ <do_not_open_new_file_init>N</do_not_open_new_file_init>
+ <extention>txt</extention>
+ <append>N</append>
+ <split>N</split>
+ <haspartno>N</haspartno>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <SpecifyFormat>N</SpecifyFormat>
+ <date_time_format/>
+ <add_to_result_filenames>Y</add_to_result_filenames>
+ <pad>N</pad>
+ <fast_dump>N</fast_dump>
+ <splitevery/>
+ </file>
+ <fields>
+ <field>
+ <name>marker</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>384</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/hop_server/reflection/0009-resolver.hpl
b/integration-tests/hop_server/reflection/0009-resolver.hpl
new file mode 100644
index 0000000000..a99fb49ab9
--- /dev/null
+++ b/integration-tests/hop_server/reflection/0009-resolver.hpl
@@ -0,0 +1,174 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0009-resolver</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Pipeline referenced by a Variable Resolver (pipeline)
metadata object. When the
+ resolver is used (#{resolver-0009:...}) this pipeline runs to produce
the resolved value; it
+ also writes a marker file, which the test verifies to prove the
metadata-referenced resolver
+ pipeline was bundled into the export and executed on the remote server
(#3368).</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2024/01/01 00:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2024/01/01 00:00:00.000</modified_date>
+ <key_for_session_key/>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Generate rows</from>
+ <to>Write marker</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Write marker</from>
+ <to>OUTPUT</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Generate rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>resolved</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif>resolved-value-3368</nullif>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <limit>1</limit>
+ <never_ending>N</never_ending>
+ <interval_in_ms>5000</interval_in_ms>
+ <row_time_field>now</row_time_field>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <attributes/>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Write marker</name>
+ <type>TextFileOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <separator>;</separator>
+ <enclosure>"</enclosure>
+ <enclosure_forced>N</enclosure_forced>
+ <enclosure_fix_disabled>N</enclosure_fix_disabled>
+ <header>Y</header>
+ <footer>N</footer>
+ <format>DOS</format>
+ <compression>None</compression>
+ <encoding>UTF-8</encoding>
+ <endedLine/>
+ <fileNameInField>N</fileNameInField>
+ <fileNameField/>
+ <create_parent_folder>Y</create_parent_folder>
+ <file>
+ <name>${PROJECT_HOME}/output-0009/resolver-marker</name>
+ <servlet_output>N</servlet_output>
+ <do_not_open_new_file_init>N</do_not_open_new_file_init>
+ <extention>txt</extention>
+ <append>N</append>
+ <split>N</split>
+ <haspartno>N</haspartno>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <SpecifyFormat>N</SpecifyFormat>
+ <date_time_format/>
+ <add_to_result_filenames>Y</add_to_result_filenames>
+ <pad>N</pad>
+ <fast_dump>N</fast_dump>
+ <splitevery/>
+ </file>
+ <fields>
+ <field>
+ <name>resolved</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>384</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>OUTPUT</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>592</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/plugins/misc/reflection/src/test/java/org/apache/hop/reflection/export/MetadataResourceExportTest.java
b/plugins/misc/reflection/src/test/java/org/apache/hop/reflection/export/MetadataResourceExportTest.java
new file mode 100644
index 0000000000..147b1eea4b
--- /dev/null
+++
b/plugins/misc/reflection/src/test/java/org/apache/hop/reflection/export/MetadataResourceExportTest.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.reflection.export;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.metadata.api.HopMetadata;
+import org.apache.hop.metadata.api.HopMetadataBase;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+import org.apache.hop.metadata.api.IHopMetadata;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.plugin.MetadataPluginType;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.reflection.pipeline.meta.PipelineLog;
+import org.apache.hop.reflection.pipeline.meta.PipelineToLogLocation;
+import org.apache.hop.reflection.workflow.meta.WorkflowLog;
+import org.apache.hop.resource.ResourceUtil;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Proves that pipelines referenced from metadata objects (here a {@link
PipelineLog}) are bundled
+ * into the resource-export ZIP and that their references are rewritten to
resolve inside the
+ * archive on a remote Hop Server. See issue #3368.
+ *
+ * <p>Before the fix the referenced pipelines were missing from the ZIP and
the metadata still
+ * pointed at the original local path, so the server logged "... pipeline file
couldn't be found to
+ * execute."
+ */
+class MetadataResourceExportTest {
+
+ private static final String MINIMAL_PIPELINE =
+ "<pipeline><info><name>%s</name></info>"
+ +
"<order></order><transform_error_handling></transform_error_handling></pipeline>";
+
+ @BeforeAll
+ static void init() throws Exception {
+ HopClientEnvironment.init();
+ PluginRegistry registry = PluginRegistry.getInstance();
+ registry.registerPluginClass(
+ PipelineLog.class.getName(), MetadataPluginType.class,
HopMetadata.class);
+ registry.registerPluginClass(
+ WorkflowLog.class.getName(), MetadataPluginType.class,
HopMetadata.class);
+ registry.registerPluginClass(
+ HopFileReference.class.getName(), MetadataPluginType.class,
HopMetadata.class);
+ }
+
+ /**
+ * Minimal metadata type holding a {@link HopMetadataPropertyType#HOP_FILE}
reference, used to
+ * prove the export decides pipeline-vs-workflow from the file extension.
+ */
+ @HopMetadata(key = "test-hop-file-reference", name = "Test HOP_FILE
reference")
+ public static class HopFileReference extends HopMetadataBase implements
IHopMetadata {
+ @HopMetadataProperty(hopMetadataPropertyType =
HopMetadataPropertyType.HOP_FILE)
+ private String referencedFile;
+
+ public String getReferencedFile() {
+ return referencedFile;
+ }
+
+ public void setReferencedFile(String referencedFile) {
+ this.referencedFile = referencedFile;
+ }
+ }
+
+ private File writeWorkflow(File dir, String name) throws Exception {
+ File file = new File(dir, name + ".hwf");
+ Files.write(
+ file.toPath(),
+ ("<workflow><name>" + name +
"</name><actions></actions><hops></hops></workflow>")
+ .getBytes(StandardCharsets.UTF_8));
+ return file;
+ }
+
+ private File writePipeline(File dir, String name) throws Exception {
+ File file = new File(dir, name + ".hpl");
+ Files.write(
+ file.toPath(), String.format(MINIMAL_PIPELINE,
name).getBytes(StandardCharsets.UTF_8));
+ return file;
+ }
+
+ @Test
+ void referencedPipelinesAreBundledAndRewritten(@TempDir File tempDir) throws
Exception {
+ IVariables variables = new Variables();
+
+ // Two pipelines referenced from a single Pipeline Log metadata object:
+ // - the main logging pipeline (direct String field)
+ // - an extra logging location (String field inside a
List<PipelineToLogLocation>)
+ File logPipeline = writePipeline(tempDir, "pipeline-log");
+ File extraLogPipeline = writePipeline(tempDir, "extra-log");
+ File mainFile = writePipeline(tempDir, "main");
+
+ IHopMetadataProvider provider = new MemoryMetadataProvider();
+ PipelineLog pipelineLog = new PipelineLog("test-log");
+ pipelineLog.setEnabled(true);
+ pipelineLog.setPipelineFilename(logPipeline.getAbsolutePath());
+ pipelineLog
+ .getPipelinesToLog()
+ .add(new PipelineToLogLocation(extraLogPipeline.getAbsolutePath()));
+ provider.getSerializer(PipelineLog.class).save(pipelineLog);
+
+ PipelineMeta mainMeta = new PipelineMeta(mainFile.getAbsolutePath(),
provider, variables);
+
+ File zip = new File(tempDir, "export.zip");
+ ResourceUtil.serializeResourceExportInterface(
+ zip.getAbsolutePath(),
+ mainMeta,
+ variables,
+ provider,
+ null,
+ null,
+ null,
+ null,
+ new HashMap<>());
+
+ List<String> entries = new ArrayList<>();
+ String metadataJson = null;
+ try (ZipInputStream zis = new
ZipInputStream(Files.newInputStream(zip.toPath()))) {
+ ZipEntry e;
+ while ((e = zis.getNextEntry()) != null) {
+ entries.add(e.getName());
+ if ("metadata.json".equals(e.getName())) {
+ metadataJson = new String(zis.readAllBytes(),
StandardCharsets.UTF_8);
+ }
+ }
+ }
+
+ // The main pipeline plus BOTH referenced logging pipelines must be
bundled in the ZIP.
+ long hplCount = entries.stream().filter(name ->
name.endsWith(".hpl")).count();
+ assertEquals(3, hplCount, "Expected main + 2 referenced pipelines in the
ZIP, got: " + entries);
+
+ // The metadata references must be rewritten to resolve inside the archive
on the server,
+ // not left pointing at the original local paths.
+ assertFalse(
+ metadataJson.contains(logPipeline.getAbsolutePath()),
+ "metadata.json still contains the original pipeline path: " +
metadataJson);
+ assertFalse(
+ metadataJson.contains(extraLogPipeline.getAbsolutePath()),
+ "metadata.json still contains the original list pipeline path: " +
metadataJson);
+ // (JSON escapes '/' as '\/', so match the variable name only.)
+ assertTrue(
+ metadataJson.contains("${Internal.Entry.Current.Folder}"),
+ "metadata.json references were not rewritten to the archive folder: "
+ metadataJson);
+ }
+
+ @Test
+ void referencedWorkflowIsBundledAndRewritten(@TempDir File tempDir) throws
Exception {
+ IVariables variables = new Variables();
+
+ // A Workflow Log references a pipeline (PIPELINE_FILE String) and a
workflow
+ // (WORKFLOW_FILE inside a List<String>).
+ File logPipeline = writePipeline(tempDir, "wf-log-pipeline");
+ File loggedWorkflow = writeWorkflow(tempDir, "logged-workflow");
+ File mainFile = writePipeline(tempDir, "main");
+
+ IHopMetadataProvider provider = new MemoryMetadataProvider();
+ WorkflowLog workflowLog = new WorkflowLog("test-workflow-log");
+ workflowLog.setEnabled(true);
+ workflowLog.setPipelineFilename(logPipeline.getAbsolutePath());
+ workflowLog.getWorkflowToLog().add(loggedWorkflow.getAbsolutePath());
+ provider.getSerializer(WorkflowLog.class).save(workflowLog);
+
+ PipelineMeta mainMeta = new PipelineMeta(mainFile.getAbsolutePath(),
provider, variables);
+
+ File zip = new File(tempDir, "export.zip");
+ ResourceUtil.serializeResourceExportInterface(
+ zip.getAbsolutePath(),
+ mainMeta,
+ variables,
+ provider,
+ null,
+ null,
+ null,
+ null,
+ new HashMap<>());
+
+ List<String> entries = new ArrayList<>();
+ String metadataJson = null;
+ try (ZipInputStream zis = new
ZipInputStream(Files.newInputStream(zip.toPath()))) {
+ ZipEntry e;
+ while ((e = zis.getNextEntry()) != null) {
+ entries.add(e.getName());
+ if ("metadata.json".equals(e.getName())) {
+ metadataJson = new String(zis.readAllBytes(),
StandardCharsets.UTF_8);
+ }
+ }
+ }
+
+ // The referenced pipeline must be bundled (main + log pipeline) and the
workflow too.
+ assertEquals(
+ 2,
+ entries.stream().filter(name -> name.endsWith(".hpl")).count(),
+ "Expected main + referenced log pipeline in the ZIP, got: " + entries);
+ assertEquals(
+ 1,
+ entries.stream().filter(name -> name.endsWith(".hwf")).count(),
+ "Expected the referenced workflow to be bundled, got: " + entries);
+
+ assertFalse(
+ metadataJson.contains(logPipeline.getAbsolutePath()),
+ "metadata.json still contains the original pipeline path: " +
metadataJson);
+ assertFalse(
+ metadataJson.contains(loggedWorkflow.getAbsolutePath()),
+ "metadata.json still contains the original workflow path: " +
metadataJson);
+ assertTrue(
+ metadataJson.contains("${Internal.Entry.Current.Folder}"),
+ "metadata.json references were not rewritten to the archive folder: "
+ metadataJson);
+ }
+
+ @Test
+ void hopFileReferenceBundlesPipelineOrWorkflowByExtension(@TempDir File
tempDir)
+ throws Exception {
+ IVariables variables = new Variables();
+
+ File referencedPipeline = writePipeline(tempDir, "hopfile-pipeline");
+ File referencedWorkflow = writeWorkflow(tempDir, "hopfile-workflow");
+ File mainFile = writePipeline(tempDir, "main");
+
+ IHopMetadataProvider provider = new MemoryMetadataProvider();
+ HopFileReference pipelineRef = new HopFileReference();
+ pipelineRef.setName("hopfile-pipeline-ref");
+ pipelineRef.setReferencedFile(referencedPipeline.getAbsolutePath());
+ provider.getSerializer(HopFileReference.class).save(pipelineRef);
+ HopFileReference workflowRef = new HopFileReference();
+ workflowRef.setName("hopfile-workflow-ref");
+ workflowRef.setReferencedFile(referencedWorkflow.getAbsolutePath());
+ provider.getSerializer(HopFileReference.class).save(workflowRef);
+
+ PipelineMeta mainMeta = new PipelineMeta(mainFile.getAbsolutePath(),
provider, variables);
+
+ File zip = new File(tempDir, "export.zip");
+ ResourceUtil.serializeResourceExportInterface(
+ zip.getAbsolutePath(),
+ mainMeta,
+ variables,
+ provider,
+ null,
+ null,
+ null,
+ null,
+ new HashMap<>());
+
+ List<String> entries = new ArrayList<>();
+ String metadataJson = null;
+ try (ZipInputStream zis = new
ZipInputStream(Files.newInputStream(zip.toPath()))) {
+ ZipEntry e;
+ while ((e = zis.getNextEntry()) != null) {
+ entries.add(e.getName());
+ if ("metadata.json".equals(e.getName())) {
+ metadataJson = new String(zis.readAllBytes(),
StandardCharsets.UTF_8);
+ }
+ }
+ }
+
+ // The .hpl HOP_FILE is bundled as a pipeline (main + it), the .hwf as a
workflow.
+ assertEquals(
+ 2,
+ entries.stream().filter(name -> name.endsWith(".hpl")).count(),
+ "Expected main + referenced pipeline in the ZIP, got: " + entries);
+ assertEquals(
+ 1,
+ entries.stream().filter(name -> name.endsWith(".hwf")).count(),
+ "Expected the referenced workflow to be bundled, got: " + entries);
+ assertFalse(
+ metadataJson.contains(referencedPipeline.getAbsolutePath()),
+ "metadata.json still contains the original HOP_FILE pipeline path: " +
metadataJson);
+ assertFalse(
+ metadataJson.contains(referencedWorkflow.getAbsolutePath()),
+ "metadata.json still contains the original HOP_FILE workflow path: " +
metadataJson);
+ }
+}
diff --git
a/plugins/transforms/kafka/src/main/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInputMeta.java
b/plugins/transforms/kafka/src/main/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInputMeta.java
index 6ee54fe7e3..c596d463d6 100644
---
a/plugins/transforms/kafka/src/main/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInputMeta.java
+++
b/plugins/transforms/kafka/src/main/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInputMeta.java
@@ -454,6 +454,18 @@ public class KafkaConsumerInputMeta
return TransformWithMappingMeta.loadMappingMeta(this, metadataProvider,
variables);
}
+ /**
+ * The sub-pipeline path is kept in this transform's own {@code filename}
field (serialized as
+ * {@code pipelinePath}), which shadows the one in {@link
TransformWithMappingMeta}. Resource
+ * export rewrites the reference through this method, so it must update the
shadowing field -
+ * otherwise the exported {@code pipelinePath} keeps pointing at the
original location and the
+ * bundled sub-pipeline can't be found on a remote server (see the analysis
for #3368).
+ */
+ @Override
+ public void replaceFileName(String fileName) {
+ this.filename = fileName;
+ }
+
@Override
public boolean supportsErrorHandling() {
return true;
diff --git
a/plugins/transforms/kafka/src/test/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInputMetaTest.java
b/plugins/transforms/kafka/src/test/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInputMetaTest.java
index 4ab6a3d853..7dd6596728 100644
---
a/plugins/transforms/kafka/src/test/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInputMetaTest.java
+++
b/plugins/transforms/kafka/src/test/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInputMetaTest.java
@@ -253,4 +253,22 @@ class KafkaConsumerInputMetaTest {
assertTrue(copy.isStopWhenIdle());
assertEquals("2500", copy.getMaxIdleTimeMs());
}
+
+ /**
+ * On resource export {@code TransformWithMappingMeta.exportResources}
bundles the sub-pipeline
+ * and rewrites the reference through {@code replaceFileName(...)}. The
sub-pipeline path is
+ * stored in this transform's own {@code filename} field (serialized as
{@code pipelinePath}) and
+ * read back through {@link KafkaConsumerInputMeta#getFilename()}, so the
rewrite must land on
+ * that same field, otherwise the exported reference still points at the
original path and the
+ * sub-pipeline can't be found on a remote server.
+ */
+ @Test
+ void replaceFileNameUpdatesThePipelinePathReference() {
+ KafkaConsumerInputMeta meta = new KafkaConsumerInputMeta();
+ meta.setFilename("${PROJECT_HOME}/sub-pipeline.hpl");
+
+ meta.replaceFileName("${Internal.Entry.Current.Folder}/sub-pipeline.hpl");
+
+ assertEquals("${Internal.Entry.Current.Folder}/sub-pipeline.hpl",
meta.getFilename());
+ }
}
