This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new f078498c50 Serialized metadata leaves out an object it can't load
instead of failing, fixes #9540 (#9541)
f078498c50 is described below
commit f078498c50c01ecbc1741488e08488eedeb567ca
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Tue Nov 17 16:14:47 2026 +0100
Serialized metadata leaves out an object it can't load instead of failing,
fixes #9540 (#9541)
---
.../metadata/SerializableMetadataProvider.java | 64 ++++++++++++++++++++--
.../metadata/SerializableMetadataProviderTest.java | 51 +++++++++++++++++
2 files changed, 109 insertions(+), 6 deletions(-)
diff --git
a/engine/src/main/java/org/apache/hop/core/metadata/SerializableMetadataProvider.java
b/engine/src/main/java/org/apache/hop/core/metadata/SerializableMetadataProvider.java
index 2f37fc57f4..a4ebbbacf6 100644
---
a/engine/src/main/java/org/apache/hop/core/metadata/SerializableMetadataProvider.java
+++
b/engine/src/main/java/org/apache/hop/core/metadata/SerializableMetadataProvider.java
@@ -18,19 +18,25 @@
package org.apache.hop.core.metadata;
import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.logging.HopLogStore;
import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.plugins.PluginRegistry;
import org.apache.hop.metadata.api.HopMetadata;
import org.apache.hop.metadata.api.IHopMetadata;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.metadata.plugin.MetadataPluginType;
import org.apache.hop.metadata.serializer.json.JsonMetadataParser;
import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.metadata.util.HopMetadataUtil;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
@@ -134,13 +140,13 @@ public class SerializableMetadataProvider extends
MemoryMetadataProvider
public SerializableMetadataProvider(String storeJson) throws HopException {
this();
this.description = "Serializable metadata provider (source is JSON)";
- JsonFactory jsonFactory = new JsonFactory();
+ ObjectMapper objectMapper = new ObjectMapper();
+ JsonFactory jsonFactory = objectMapper.getFactory();
try {
try (ByteArrayInputStream inputStream =
new
ByteArrayInputStream(storeJson.getBytes(StandardCharsets.UTF_8));
- com.fasterxml.jackson.core.JsonParser jsonParser =
- jsonFactory.createParser(inputStream)) {
+ JsonParser jsonParser = jsonFactory.createParser(inputStream)) {
// Loop over the classes until there's no more left
//
@@ -177,9 +183,19 @@ public class SerializableMetadataProvider extends
MemoryMetadataProvider
// Loop over the metadata objects in the JSON for the given class...
//
do {
- IHopMetadata object = metadataParser.loadJsonObject(managedClass,
jsonParser);
- if (object != null) {
- serializer.save(object);
+ // One object at a time: an object that can't be built here,
because it refers to a
+ // plugin this Hop doesn't have, is left out instead of failing
everything that reads
+ // this metadata. It is read whole first, so the next object
starts where it should.
+ //
+ JsonNode objectNode = objectMapper.readTree(jsonParser);
+ try (JsonParser objectParser = objectNode.traverse(objectMapper)) {
+ objectParser.nextToken();
+ IHopMetadata object =
metadataParser.loadJsonObject(managedClass, objectParser);
+ if (object != null) {
+ serializer.save(object);
+ }
+ } catch (Exception e) {
+ logSkippedObject(classKey, objectNode, e);
}
} while (jsonParser.nextToken() != JsonToken.END_ARRAY);
}
@@ -194,6 +210,14 @@ public class SerializableMetadataProvider extends
MemoryMetadataProvider
* @return
*/
private Class<IHopMetadata> getManagedClass(String classKey) {
+ if (!isMetadataTypeInstalled(classKey)) {
+ // Metadata written by a Hop with more plugins installed: nothing here
can use it
+ LogChannel.GENERAL.logBasic(
+ "Skipping the metadata of type '"
+ + classKey
+ + "': no installed plugin provides this type.");
+ return null;
+ }
Class<IHopMetadata> managedClass;
try {
managedClass = getMetadataClassForKey(classKey);
@@ -203,4 +227,32 @@ public class SerializableMetadataProvider extends
MemoryMetadataProvider
}
return managedClass;
}
+
+ /** Whether an installed plugin provides this metadata type, under its key
or a former key. */
+ private boolean isMetadataTypeInstalled(String classKey) {
+ if
(PluginRegistry.getInstance().findPluginWithId(MetadataPluginType.class,
classKey) != null) {
+ return true;
+ }
+ for (Class<IHopMetadata> metadataClass :
this.<IHopMetadata>getMetadataClasses()) {
+ if
(HopMetadataUtil.matchesKey(metadataClass.getAnnotation(HopMetadata.class),
classKey)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** One object of an installed type that this Hop can't build: one line, the
cause included. */
+ private static void logSkippedObject(String classKey, JsonNode objectNode,
Exception e) {
+ JsonNode name = objectNode.get("name");
+ LogChannel.GENERAL.logError(
+ "Skipping the metadata object '"
+ + (name == null ? "?" : name.asText())
+ + "' of type '"
+ + classKey
+ + "': it can't be loaded here. "
+ + Const.NVL(e.getMessage(), e.toString()).trim());
+ if (LogChannel.GENERAL.isDebug()) {
+ LogChannel.GENERAL.logDebug(Const.getStackTracker(e));
+ }
+ }
}
diff --git
a/engine/src/test/java/org/apache/hop/core/metadata/SerializableMetadataProviderTest.java
b/engine/src/test/java/org/apache/hop/core/metadata/SerializableMetadataProviderTest.java
index b09ba06d3e..171c7dfeeb 100644
---
a/engine/src/test/java/org/apache/hop/core/metadata/SerializableMetadataProviderTest.java
+++
b/engine/src/test/java/org/apache/hop/core/metadata/SerializableMetadataProviderTest.java
@@ -29,6 +29,7 @@ import org.apache.hop.metadata.api.IHopMetadata;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.metadata.api.IHopMetadataSerializer;
import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.pipeline.config.PipelineRunConfiguration;
import org.apache.hop.server.HopServerMeta;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -203,6 +204,56 @@ class SerializableMetadataProviderTest {
assertServerAfterUnavailableConnection(json, "graph-database-connection");
}
+ /**
+ * An object of an installed type that refers to a plugin this Hop doesn't
have is left out; the
+ * objects around it still load.
+ */
+ @Test
+ void testObjectThatCannotBeBuiltIsSkipped() throws Exception {
+ final var json =
+ """
+ {
+ "pipeline-run-configuration": [
+ {"name": "before", "engineRunConfiguration": {"Local": {}}},
+ {"name": "unavailable", "engineRunConfiguration":
{"NotInstalledEngine": {}}},
+ {"name": "after", "engineRunConfiguration": {"Local": {}}}
+ ],
+ "server": [{"name": "server1", "hostname": "localhost"}]
+ }
+ """;
+
+ final var provider = new SerializableMetadataProvider(json);
+
+ final var runConfigurations =
provider.getSerializer(PipelineRunConfiguration.class);
+ assertEquals(
+ List.of("after", "before"),
runConfigurations.listObjectNames().stream().sorted().toList());
+
assertNotNull(runConfigurations.load("before").getEngineRunConfiguration());
+ assertNotNull(provider.getSerializer(HopServerMeta.class).load("server1"));
+ }
+
+ /** A type no plugin provides is skipped without looking up its class, which
logs an error. */
+ @Test
+ void testUnknownTypeWithObjectsIsSkippedWithoutLookup() throws Exception {
+ final var json =
+ """
+ {
+ "not-installed-type": [{"name": "x", "nested": [1, 2]}],
+ "server": [{"name": "server1", "hostname": "localhost"}]
+ }
+ """;
+ final var provider =
+ new SerializableMetadataProvider(json) {
+ @Override
+ public <T extends IHopMetadata> Class<T>
getMetadataClassForKey(final String key)
+ throws HopException {
+ assertEquals("server", key, "A type no plugin provides must not be
looked up");
+ return super.getMetadataClassForKey(key);
+ }
+ };
+
+ assertNotNull(provider.getSerializer(HopServerMeta.class).load("server1"));
+ }
+
private void assertServerAfterUnavailableConnection(
final String json, final String unavailableKey) throws Exception {
final var provider =