This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 0e6ac7303c issue #7736: merge marketplace plugin metadata on same-id
repo import (#7737)
0e6ac7303c is described below
commit 0e6ac7303c1e89f8f91909230c802158ec2e8c29
Author: Matt Casters <[email protected]>
AuthorDate: Sun Aug 2 13:46:54 2026 +0200
issue #7736: merge marketplace plugin metadata on same-id repo import
(#7737)
When multiple hop-marketplace-repo.yaml files share a repository id (e.g.
community Nexus), import merges plugins by G:A instead of replacing the
list. Treat x.y.z-SNAPSHOT as the x.y.z Hop line for min/maxHopVersion
compatibility so SNAPSHOT builds fulfill the same release requirement.
---
plugins/misc/marketplace/README.md | 9 +-
.../hop/marketplace/catalog/VersionCompat.java | 27 +++++-
.../marketplace/command/MarketplaceCommand.java | 2 +-
.../config/MarketplaceRepositoryDefinition.java | 85 +++++++++++++++-
.../hop/marketplace/catalog/VersionCompatTest.java | 22 +++++
.../MarketplaceRepositoryDefinitionTest.java | 108 +++++++++++++++++++++
6 files changed, 247 insertions(+), 6 deletions(-)
diff --git a/plugins/misc/marketplace/README.md
b/plugins/misc/marketplace/README.md
index cb88d010d4..89e54c4a9e 100644
--- a/plugins/misc/marketplace/README.md
+++ b/plugins/misc/marketplace/README.md
@@ -211,9 +211,16 @@ Three steps:
- Bundled Apache optional plugins: **version = running Hop version** (not
absolute latest on ASF)
- Every enabled repo with **`browse: true`**: live Nexus search for `*.zip`
(one row per artifact, latest plugin version, last updated)
- Optional YAML `plugins:` metadata **enriches** names/categories and can set:
- - `minHopVersion` / `maxHopVersion` — hide if this Hop is outside the range
(e.g. datavault needs ≥ 2.18.1)
+ - `minHopVersion` / `maxHopVersion` — hide if this Hop is outside the range
(e.g. datavault needs ≥ 2.18.1). A running `x.y.z-SNAPSHOT` counts as the
`x.y.z` line (so `2.19.0-SNAPSHOT` fulfills `minHopVersion: 2.19.0`)
- `version` — optional pin of the plugin artifact; otherwise latest from
browse
+**Shared repository id (multi-plugin community Nexus)**
+
+Several projects may publish their own `hop-marketplace-repo.yaml` with the
**same** `id`
+and Nexus `url` (each listing only its own plugin under `plugins:`). Import
merges
+plugin metadata by `groupId` + `artifactId` into one hop-config entry — later
imports
+do not wipe earlier plugins. Re-importing the same G:A refreshes that entry’s
fields.
+
GUI **Plugins** tab uses the same discovery as `query`.
Sample: `src/main/samples/hop-marketplace-repo.community.example.yaml`
diff --git
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/catalog/VersionCompat.java
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/catalog/VersionCompat.java
index 681fb3ec26..a7ef57f141 100644
---
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/catalog/VersionCompat.java
+++
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/catalog/VersionCompat.java
@@ -37,6 +37,11 @@ public final class VersionCompat {
/**
* Whether {@code info} may be offered on the given running Hop version.
Blank min/max means no
* bound on that side. Missing hop version fails closed only when a bound is
present.
+ *
+ * <p>{@code x.y.z-SNAPSHOT} is treated as the {@code x.y.z} line for these
bounds so developers
+ * on a SNAPSHOT build satisfy a {@code minHopVersion} of the same release
(e.g. {@code
+ * 2.19.0-SNAPSHOT} fulfills {@code minHopVersion: 2.19.0}). Artifact
version ordering via {@link
+ * #compare(String, String)} still ranks SNAPSHOT below the release.
*/
public static boolean isCompatibleWithHop(OptionalPluginInfo info, String
hopVersion) {
if (info == null) {
@@ -50,15 +55,33 @@ public final class VersionCompat {
if (StringUtils.isBlank(hopVersion)) {
return false;
}
- if (StringUtils.isNotBlank(min) && compare(hopVersion, min) < 0) {
+ // Compatibility uses the release line; SNAPSHOT is development of that
line.
+ String hopLine = stripSnapshotQualifier(hopVersion);
+ if (StringUtils.isNotBlank(min) && compare(hopLine,
stripSnapshotQualifier(min)) < 0) {
return false;
}
- if (StringUtils.isNotBlank(max) && compare(hopVersion, max) > 0) {
+ if (StringUtils.isNotBlank(max) && compare(hopLine,
stripSnapshotQualifier(max)) > 0) {
return false;
}
return true;
}
+ /**
+ * Strip a trailing {@code -SNAPSHOT} qualifier (case-insensitive). Used for
Hop min/max
+ * compatibility only.
+ */
+ static String stripSnapshotQualifier(String version) {
+ if (StringUtils.isBlank(version)) {
+ return version;
+ }
+ String v = version.trim();
+ String lower = v.toLowerCase(Locale.ROOT);
+ if (lower.endsWith("-snapshot")) {
+ return v.substring(0, v.length() - "-snapshot".length());
+ }
+ return v;
+ }
+
/**
* @return negative if a < b, zero if equal, positive if a > b
*/
diff --git
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/command/MarketplaceCommand.java
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/command/MarketplaceCommand.java
index 88b09986b6..074d8fc04a 100644
---
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/command/MarketplaceCommand.java
+++
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/command/MarketplaceCommand.java
@@ -718,7 +718,7 @@ public class MarketplaceCommand implements Runnable,
IHopCommand, IHasHopMetadat
name = "import",
description =
"Import a hop-marketplace-repo.yaml (or URL) into hop-config.json.
Upserts by repository"
- + " id.")
+ + " id; merges plugin metadata when the id already exists.")
static class RepoImportCommand extends MarketplaceSubCommand {
@Parameters(
index = "0",
diff --git
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/config/MarketplaceRepositoryDefinition.java
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/config/MarketplaceRepositoryDefinition.java
index 625f923f0e..88f4175679 100644
---
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/config/MarketplaceRepositoryDefinition.java
+++
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/config/MarketplaceRepositoryDefinition.java
@@ -347,13 +347,94 @@ public final class MarketplaceRepositoryDefinition {
existing.setGroupIdFilter(imported.getGroupIdFilter());
existing.setHomepage(imported.getHomepage());
existing.setDescription(imported.getDescription());
- if (imported.getPlugins() != null) {
- existing.setPlugins(new ArrayList<>(imported.getPlugins()));
+ // Multiple per-plugin YAMLs may share one repository id (e.g. community
Nexus).
+ // Merge plugin metadata by G:A; empty import must not wipe existing
entries.
+ if (imported.getPlugins() != null && !imported.getPlugins().isEmpty()) {
+ existing.setPlugins(mergePlugins(existing.getPlugins(),
imported.getPlugins()));
}
config.ensureValidPrimary();
}
}
+ /**
+ * Merge plugin metadata lists for same-id repository import.
+ *
+ * <p>Existing entries not present in {@code imported} are kept. Imported
entries with a matching
+ * G:A (or artifactId when groupId is missing on either side) replace the
prior entry so re-import
+ * refreshes version and display fields. New G:A values are appended. Order:
prior plugins first,
+ * then newly appended imports.
+ */
+ static List<OptionalPluginInfo> mergePlugins(
+ List<OptionalPluginInfo> existing, List<OptionalPluginInfo> imported) {
+ List<OptionalPluginInfo> result = new ArrayList<>();
+ if (existing != null) {
+ for (OptionalPluginInfo p : existing) {
+ if (p != null && StringUtils.isNotBlank(p.getArtifactId())) {
+ result.add(p);
+ }
+ }
+ }
+ if (imported == null || imported.isEmpty()) {
+ return result;
+ }
+ for (OptionalPluginInfo incoming : imported) {
+ if (incoming == null || StringUtils.isBlank(incoming.getArtifactId())) {
+ continue;
+ }
+ int idx = indexOfMatchingPlugin(result, incoming);
+ if (idx >= 0) {
+ result.set(idx, incoming);
+ } else {
+ result.add(incoming);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Find an existing plugin that should be updated by {@code candidate}:
prefer groupId+artifactId,
+ * fall back to artifactId alone when either side lacks groupId.
+ */
+ static int indexOfMatchingPlugin(List<OptionalPluginInfo> plugins,
OptionalPluginInfo candidate) {
+ if (plugins == null || candidate == null ||
StringUtils.isBlank(candidate.getArtifactId())) {
+ return -1;
+ }
+ String candArt = candidate.getArtifactId().toLowerCase(Locale.ROOT);
+ String candGa = pluginGaKey(candidate);
+ boolean candHasGroup = StringUtils.isNotBlank(candidate.getGroupId());
+
+ for (int i = 0; i < plugins.size(); i++) {
+ OptionalPluginInfo p = plugins.get(i);
+ if (p == null || StringUtils.isBlank(p.getArtifactId())) {
+ continue;
+ }
+ if (!candArt.equals(p.getArtifactId().toLowerCase(Locale.ROOT))) {
+ continue;
+ }
+ boolean pHasGroup = StringUtils.isNotBlank(p.getGroupId());
+ if (candHasGroup && pHasGroup) {
+ if (candGa.equals(pluginGaKey(p))) {
+ return i;
+ }
+ // Same artifactId, different groupId → distinct plugins
+ continue;
+ }
+ // One or both sides lack groupId: treat as the same plugin
+ return i;
+ }
+ return -1;
+ }
+
+ /** Lowercase {@code groupId:artifactId}; blank groupId becomes empty
prefix. */
+ static String pluginGaKey(OptionalPluginInfo info) {
+ if (info == null || StringUtils.isBlank(info.getArtifactId())) {
+ return "";
+ }
+ String g =
+ StringUtils.isNotBlank(info.getGroupId()) ?
info.getGroupId().toLowerCase(Locale.ROOT) : "";
+ return g + ":" + info.getArtifactId().toLowerCase(Locale.ROOT);
+ }
+
private static String stringVal(Object o) {
if (o == null) {
return null;
diff --git
a/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/catalog/VersionCompatTest.java
b/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/catalog/VersionCompatTest.java
index e0b50568ba..8f7c002df2 100644
---
a/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/catalog/VersionCompatTest.java
+++
b/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/catalog/VersionCompatTest.java
@@ -50,12 +50,34 @@ class VersionCompatTest {
assertFalse(VersionCompat.isCompatibleWithHop(info, "2.17.0"));
}
+ @Test
+ void minHopVersionSnapshotMatchesSameReleaseLine() {
+ OptionalPluginInfo info = new OptionalPluginInfo();
+ info.setArtifactId("hop-pentaho-reporting-output");
+ info.setMinHopVersion("2.19.0");
+ // Developers on 2.19.0-SNAPSHOT are on the 2.19.0 line
+ assertTrue(VersionCompat.isCompatibleWithHop(info, "2.19.0-SNAPSHOT"));
+ assertTrue(VersionCompat.isCompatibleWithHop(info, "2.19.0"));
+ assertTrue(VersionCompat.isCompatibleWithHop(info, "2.19.1-SNAPSHOT"));
+ assertFalse(VersionCompat.isCompatibleWithHop(info, "2.18.0-SNAPSHOT"));
+ assertFalse(VersionCompat.isCompatibleWithHop(info, "2.18.1"));
+ }
+
@Test
void maxHopVersionFilter() {
OptionalPluginInfo info = new OptionalPluginInfo();
info.setMaxHopVersion("2.19.0");
assertTrue(VersionCompat.isCompatibleWithHop(info, "2.18.0"));
+ assertTrue(VersionCompat.isCompatibleWithHop(info, "2.19.0-SNAPSHOT"));
assertFalse(VersionCompat.isCompatibleWithHop(info, "2.20.0"));
+ assertFalse(VersionCompat.isCompatibleWithHop(info, "2.20.0-SNAPSHOT"));
+ }
+
+ @Test
+ void compareStillRanksSnapshotBelowRelease() {
+ // Artifact ordering (latest / preferable) must keep SNAPSHOT < same-base
release
+ assertTrue(VersionCompat.compare("2.19.0-SNAPSHOT", "2.19.0") < 0);
+ assertTrue(VersionCompat.compare("2.19.0", "2.19.0-SNAPSHOT") > 0);
}
@Test
diff --git
a/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/config/MarketplaceRepositoryDefinitionTest.java
b/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/config/MarketplaceRepositoryDefinitionTest.java
index 8c728dff59..97f8963b72 100644
---
a/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/config/MarketplaceRepositoryDefinitionTest.java
+++
b/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/config/MarketplaceRepositoryDefinitionTest.java
@@ -24,6 +24,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.List;
+import org.apache.hop.marketplace.catalog.OptionalPluginInfo;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -112,4 +114,110 @@ class MarketplaceRepositoryDefinitionTest {
assertTrue(saved.contains("hop-datavault"));
assertTrue(saved.contains("minHopVersion"));
}
+
+ @Test
+ void applyToConfigMergesPluginsFromSameIdImports() throws Exception {
+ MarketplaceConfig config = new MarketplaceConfig();
+
+ MarketplaceRepository dataVault = new MarketplaceRepository();
+ dataVault.setId("data-hopper-community");
+ dataVault.setName("Data Hopper Community Plugins");
+
dataVault.setUrl("https://repository.data-hopper.com/repository/hop-community-plugins/";);
+ dataVault.setBrowse(true);
+ dataVault.setPlugins(List.of(plugin("org.apache.hop", "hop-datavault",
"0.5.0", "2.18.1")));
+ MarketplaceRepositoryDefinition.applyToConfig(config, dataVault, false);
+
+ MarketplaceRepository pentaho = new MarketplaceRepository();
+ pentaho.setId("data-hopper-community");
+ pentaho.setName("Data Hopper Community Plugins");
+
pentaho.setUrl("https://repository.data-hopper.com/repository/hop-community-plugins/";);
+ pentaho.setBrowse(true);
+ pentaho.setPlugins(
+ List.of(
+ plugin(
+ "org.projectdatahopper.hop", "hop-pentaho-reporting-output",
"1.0.0", "2.19.0")));
+ MarketplaceRepositoryDefinition.applyToConfig(config, pentaho, false);
+
+ MarketplaceRepository found =
config.findRepository("data-hopper-community");
+ assertEquals(2, found.getPlugins().size());
+ assertEquals("hop-datavault", found.getPlugins().get(0).getArtifactId());
+ assertEquals("hop-pentaho-reporting-output",
found.getPlugins().get(1).getArtifactId());
+ }
+
+ @Test
+ void applyToConfigReimportUpdatesMatchingPluginKeepsOthers() throws
Exception {
+ MarketplaceConfig config = new MarketplaceConfig();
+
+ MarketplaceRepository first = new MarketplaceRepository();
+ first.setId("community");
+ first.setUrl("https://example.com/repository/hop/";);
+ first.setPlugins(
+ List.of(
+ plugin("org.apache.hop", "hop-datavault", "0.4.0", "2.18.1"),
+ plugin("org.example", "other-plugin", "1.0.0", null)));
+ MarketplaceRepositoryDefinition.applyToConfig(config, first, false);
+
+ MarketplaceRepository update = new MarketplaceRepository();
+ update.setId("community");
+ update.setUrl("https://example.com/repository/hop/";);
+ OptionalPluginInfo refreshed = plugin("org.apache.hop", "hop-datavault",
"0.5.0", "2.19.0");
+ refreshed.setName("Data Vault");
+ update.setPlugins(List.of(refreshed));
+ MarketplaceRepositoryDefinition.applyToConfig(config, update, false);
+
+ MarketplaceRepository found = config.findRepository("community");
+ assertEquals(2, found.getPlugins().size());
+ assertEquals("0.5.0", found.getPlugins().get(0).getVersion());
+ assertEquals("2.19.0", found.getPlugins().get(0).getMinHopVersion());
+ assertEquals("Data Vault", found.getPlugins().get(0).getName());
+ assertEquals("other-plugin", found.getPlugins().get(1).getArtifactId());
+ }
+
+ @Test
+ void applyToConfigEmptyPluginsDoesNotWipeExisting() throws Exception {
+ MarketplaceConfig config = new MarketplaceConfig();
+
+ MarketplaceRepository withPlugins = new MarketplaceRepository();
+ withPlugins.setId("community");
+ withPlugins.setUrl("https://example.com/repository/hop/";);
+ withPlugins.setPlugins(List.of(plugin("org.apache.hop", "hop-datavault",
"0.5.0", null)));
+ MarketplaceRepositoryDefinition.applyToConfig(config, withPlugins, false);
+
+ MarketplaceRepository noPlugins = new MarketplaceRepository();
+ noPlugins.setId("community");
+ noPlugins.setUrl("https://example.com/repository/hop-v2/";);
+ noPlugins.setBrowse(true);
+ noPlugins.setPlugins(List.of());
+ MarketplaceRepositoryDefinition.applyToConfig(config, noPlugins, false);
+
+ MarketplaceRepository found = config.findRepository("community");
+ assertEquals("https://example.com/repository/hop-v2/";, found.getUrl());
+ assertTrue(found.isBrowse());
+ assertEquals(1, found.getPlugins().size());
+ assertEquals("hop-datavault", found.getPlugins().get(0).getArtifactId());
+ }
+
+ @Test
+ void mergePluginsMatchesArtifactIdWhenGroupIdMissing() {
+ OptionalPluginInfo existing = plugin(null, "hop-datavault", "0.4.0", null);
+ OptionalPluginInfo incoming = plugin("org.apache.hop", "hop-datavault",
"0.5.0", "2.18.1");
+
+ List<OptionalPluginInfo> merged =
+ MarketplaceRepositoryDefinition.mergePlugins(List.of(existing),
List.of(incoming));
+
+ assertEquals(1, merged.size());
+ assertEquals("0.5.0", merged.get(0).getVersion());
+ assertEquals("org.apache.hop", merged.get(0).getGroupId());
+ }
+
+ private static OptionalPluginInfo plugin(
+ String groupId, String artifactId, String version, String minHopVersion)
{
+ OptionalPluginInfo info = new OptionalPluginInfo();
+ info.setGroupId(groupId);
+ info.setArtifactId(artifactId);
+ info.setVersion(version);
+ info.setMinHopVersion(minHopVersion);
+ info.setName(artifactId);
+ return info;
+ }
}
