This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new d50979f897 Execution information locations keep their type when saved
again, fixes #9505 (#9506)
d50979f897 is described below
commit d50979f8972f1996099b900b599153a66f2d78cf
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Mon Nov 16 20:49:11 2026 +0100
Execution information locations keep their type when saved again, fixes
#9505 (#9506)
---
.../hop/execution/IExecutionInfoLocation.java | 7 +-
.../hop/core/metadata/MetadataObjectIdTest.java | 227 +++++++++++++++++++++
2 files changed, 233 insertions(+), 1 deletion(-)
diff --git
a/engine/src/main/java/org/apache/hop/execution/IExecutionInfoLocation.java
b/engine/src/main/java/org/apache/hop/execution/IExecutionInfoLocation.java
index 9bb2fefb71..03608dd277 100644
--- a/engine/src/main/java/org/apache/hop/execution/IExecutionInfoLocation.java
+++ b/engine/src/main/java/org/apache/hop/execution/IExecutionInfoLocation.java
@@ -302,7 +302,12 @@ public interface IExecutionInfoLocation extends Cloneable {
public Object createObject(String id, Object parentObject) throws
HopException {
PluginRegistry registry = PluginRegistry.getInstance();
IPlugin plugin =
registry.findPluginWithId(ExecutionInfoLocationPluginType.class, id);
- return registry.loadClass(plugin);
+ IExecutionInfoLocation location = registry.loadClass(plugin,
IExecutionInfoLocation.class);
+ // The ID is the key the location is stored under, not always a field of
its own: without
+ // it the location can't be saved again
+ location.setPluginId(plugin.getIds()[0]);
+ location.setPluginName(plugin.getName());
+ return location;
}
@Override
diff --git
a/engine/src/test/java/org/apache/hop/core/metadata/MetadataObjectIdTest.java
b/engine/src/test/java/org/apache/hop/core/metadata/MetadataObjectIdTest.java
new file mode 100644
index 0000000000..c15f7cb432
--- /dev/null
+++
b/engine/src/test/java/org/apache/hop/core/metadata/MetadataObjectIdTest.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.metadata;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.lang.reflect.ParameterizedType;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.hop.core.database.DatabasePluginType;
+import org.apache.hop.core.database.IDatabase;
+import org.apache.hop.core.plugins.IPlugin;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.execution.ExecutionInfoLocation;
+import org.apache.hop.execution.local.FileExecutionInfoLocation;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.metadata.api.HopMetadata;
+import org.apache.hop.metadata.api.HopMetadataDefaultObjectFactory;
+import org.apache.hop.metadata.api.HopMetadataObject;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.IHopMetadata;
+import org.apache.hop.metadata.api.IHopMetadataObjectFactory;
+import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.metadata.plugin.MetadataPluginType;
+import org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration;
+import org.apache.hop.pipeline.engine.PipelineEnginePluginType;
+import org.apache.hop.workflow.config.IWorkflowEngineRunConfiguration;
+import org.apache.hop.workflow.engine.WorkflowEnginePluginType;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/**
+ * An object that a metadata property creates through an object factory (an
execution information
+ * location, a sampler, a run configuration, ...) is saved under its ID, and
created again from that
+ * ID when it is read. Read and saved again, as in a metadata export or the
metadata that remote and
+ * distributed engines send their workers, every registered plugin must come
back as itself.
+ */
+@ExtendWith(RestoreHopEngineEnvironmentExtension.class)
+class MetadataObjectIdTest {
+
+ /** As the location files of the Neo4j integration tests: the plugin ID is
only the key. */
+ @Test
+ void aLocationStoredOnlyUnderItsKeyKeepsItsTypeWhenSavedAgain() throws
Exception {
+ String key =
ExecutionInfoLocation.class.getAnnotation(HopMetadata.class).key();
+ String exported =
+ "{\""
+ + key
+ +
"\":[{\"name\":\"audit\",\"dataLoggingDelay\":\"2000\",\"dataLoggingInterval\":\"5000\","
+ +
"\"executionInfoLocation\":{\"local-folder\":{\"rootFolder\":\"/tmp/executions\"}}}]}";
+
+ String exportedAgain = new SerializableMetadataProvider(exported).toJson();
+ ExecutionInfoLocation location =
+ new SerializableMetadataProvider(exportedAgain)
+ .getSerializer(ExecutionInfoLocation.class)
+ .load("audit");
+
+ FileExecutionInfoLocation fileLocation =
+ assertInstanceOf(FileExecutionInfoLocation.class,
location.getExecutionInfoLocation());
+ assertEquals("local-folder", fileLocation.getPluginId());
+ assertEquals("/tmp/executions", fileLocation.getRootFolder());
+ }
+
+ @Test
+ void everyRegisteredPluginComesBackAsItselfWhenSavedAgain() throws Exception
{
+ PluginRegistry registry = PluginRegistry.getInstance();
+ List<String> failures = new ArrayList<>();
+ Map<String, Integer> roundTrips = new TreeMap<>();
+
+ for (IPlugin metadataPlugin :
registry.getPlugins(MetadataPluginType.class)) {
+ Class<?> holderClass = registry.loadClass(metadataPlugin).getClass();
+ for (Field field : factoryFields(holderClass)) {
+ Class<?> valueType = valueType(field);
+ IHopMetadataObjectFactory factory =
+ valueType
+ .getAnnotation(HopMetadataObject.class)
+ .objectFactory()
+ .getDeclaredConstructor()
+ .newInstance();
+ String property = holderClass.getSimpleName() + "." + field.getName();
+ for (String id : pluginIds(registry, valueType)) {
+ // As read from a file, and as the copy an editor or a run works on
+ Object read = factory.createObject(id, null);
+ List<Object> objects = new ArrayList<>(List.of(read));
+ Method clone = publicClone(read);
+ if (clone != null) {
+ objects.add(clone.invoke(read));
+ }
+ for (Object object : objects) {
+ roundTrips.merge(property, 1, Integer::sum);
+ String problem = roundTrip(holderClass, field, object);
+ if (problem != null) {
+ failures.add(property + ", " + id + ": " + problem);
+ }
+ }
+ }
+ }
+ }
+
+ assertTrue(failures.isEmpty(), String.join("\n", failures));
+ // The plugins of the engine module itself are covered
+ for (String property :
+ List.of(
+ "ExecutionInfoLocation.executionInfoLocation",
+ "ExecutionDataProfile.samplers",
+ "PipelineRunConfiguration.engineRunConfiguration",
+ "WorkflowRunConfiguration.engineRunConfiguration")) {
+ assertTrue(roundTrips.getOrDefault(property, 0) > 0, property + " not
tested: " + roundTrips);
+ }
+ }
+
+ /** The properties holding an object that a factory of its own saves and
creates. */
+ private static List<Field> factoryFields(Class<?> holderClass) {
+ List<Field> fields = new ArrayList<>();
+ for (Class<?> c = holderClass; c != null && c != Object.class; c =
c.getSuperclass()) {
+ for (Field field : c.getDeclaredFields()) {
+ if (field.getAnnotation(HopMetadataProperty.class) == null) {
+ continue;
+ }
+ Class<?> valueType = valueType(field);
+ HopMetadataObject annotation =
+ valueType == null ? null :
valueType.getAnnotation(HopMetadataObject.class);
+ if (annotation != null
+ && annotation.objectFactory() !=
HopMetadataDefaultObjectFactory.class) {
+ fields.add(field);
+ }
+ }
+ }
+ return fields;
+ }
+
+ /** The type of the field, or of the elements of a list. */
+ private static Class<?> valueType(Field field) {
+ if (!List.class.isAssignableFrom(field.getType())) {
+ return field.getType();
+ }
+ if (field.getGenericType() instanceof ParameterizedType type
+ && type.getActualTypeArguments()[0] instanceof Class<?> elementType) {
+ return elementType;
+ }
+ return null;
+ }
+
+ /** The IDs of the registered plugins the factory creates objects of the
type for. */
+ private static List<String> pluginIds(PluginRegistry registry, Class<?>
valueType) {
+ List<String> ids = new ArrayList<>();
+ for (var pluginType : registry.getPluginTypes()) {
+ boolean matches;
+ if (valueType == IPipelineEngineRunConfiguration.class) {
+ matches = pluginType == PipelineEnginePluginType.class;
+ } else if (valueType == IWorkflowEngineRunConfiguration.class) {
+ matches = pluginType == WorkflowEnginePluginType.class;
+ } else if (valueType == IDatabase.class) {
+ // The database plugin type doesn't name its main type
+ matches = pluginType == DatabasePluginType.class;
+ } else {
+ matches = false;
+ }
+ for (IPlugin plugin : registry.getPlugins(pluginType)) {
+ if (matches || plugin.getMainType() == valueType) {
+ ids.add(plugin.getIds()[0]);
+ }
+ }
+ }
+ return ids;
+ }
+
+ private static Method publicClone(Object object) {
+ try {
+ return object.getClass().getMethod("clone");
+ } catch (NoSuchMethodException e) {
+ return null;
+ }
+ }
+
+ /** Null when the object is read back as itself from the JSON it was saved
in. */
+ @SuppressWarnings("unchecked")
+ private static String roundTrip(Class<?> holderClass, Field field, Object
object) {
+ try {
+ IHopMetadata holder = (IHopMetadata)
holderClass.getDeclaredConstructor().newInstance();
+ holder.setName("probe");
+ field.setAccessible(true);
+ boolean list = List.class.isAssignableFrom(field.getType());
+ field.set(holder, list ? new ArrayList<>(List.of(object)) : object);
+
+ SerializableMetadataProvider saved = new SerializableMetadataProvider();
+ ((IHopMetadataSerializer<IHopMetadata>)
+ saved.getSerializer((Class<IHopMetadata>) holderClass))
+ .save(holder);
+ SerializableMetadataProvider read = new
SerializableMetadataProvider(saved.toJson());
+ Object loaded = read.getSerializer((Class<IHopMetadata>)
holderClass).load("probe");
+
+ Object value = loaded == null ? null : field.get(loaded);
+ if (list && value != null) {
+ value = ((List<?>) value).isEmpty() ? null : ((List<?>) value).get(0);
+ }
+ if (value == null) {
+ return "not read back";
+ }
+ return value.getClass() == object.getClass()
+ ? null
+ : "read back as " + value.getClass().getName();
+ } catch (Exception e) {
+ return e.toString();
+ }
+ }
+}