This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 81c1502067 Issue #8755 : Fix the integration test failures from build
2337 (#8854)
81c1502067 is described below
commit 81c1502067df475e824ebe1e98723ab25bc94fd6
Author: Matt Casters <[email protected]>
AuthorDate: Sun Oct 11 13:42:09 2026 +0200
Issue #8755 : Fix the integration test failures from build 2337 (#8854)
* Issue #8755 : Fix the integration test failures from build 2337
The native Spark lake tests could not load LakeTableOutputMeta because the
lakehouse plugin is not in the client and was not installed with the other
marketplace plugins. PostgreSQL Bulk Loader still saw a second driver on
lib/core: the Beam assembly exclude does not apply to Debezium's transitive
driver, and the AGE plugin copied its provided driver into lib/core. Load
balancing treated a slow status probe as a missing Hop server. The Atlas
service was started on the mongo:8 hostname and marked healthy during its
init restart. The samples runner still only skipped /neo4j/, so the new
graph-databases samples ran without a database.
* Issue #8755 : Load the AGE PostgreSQL driver from the database plugin,
skip samples that need services
Graph database plugins do not see lib/jdbc, so with the driver gone from
lib/core every AGE connection failed with "No suitable driver". AGE now takes
the driver from the PostgreSQL database plugin, which honours
HOP_SHARED_JDBC_FOLDERS (Hop Web uses jdbc-drivers). The samples run also
stopped on the property-graph and extract-graph samples (missing oracle-23ai
and ollama-local metadata) and hung on watch-files.hpl, which runs until it is
stopped.
* Issue #8755 : Give the AGE tests the PostgreSQL database plugin
AgeGraphDatabase loads the JDBC driver from the PostgreSQL database plugin.
The AGE tests never had that plugin on the classpath, so AgeIT failed with
"Apache AGE needs the PostgreSQL database plugin".
---------
Co-authored-by: Bart Maertens <[email protected]>
---
.../integration-tests/integration-tests-mongo.yaml | 18 ++++++--
.../loadbalance/LoadBalancingPipelineEngine.java | 23 +++++++--
.../loadbalance/LoadBalancingCoordinator.java | 44 +++++++++++++++++-
.../loadbalance/LoadBalancingWorkflowEngine.java | 23 +++++++--
.../loadbalance/LoadBalancingCoordinatorTest.java | 21 +++++++++
.../pipeline-run-configuration/load-balance.json | 2 +-
.../workflow-run-configuration/load-balance.json | 2 +-
.../samples/read-samples-build-hop-run.hpl | 46 ++++++++++++++++++
plugins/engines/beam/pom.xml | 18 +++++++-
plugins/engines/beam/src/assembly/assembly.xml | 12 +++--
.../apache/hop/marketplace/optional-plugins.yaml | 8 ++++
plugins/tech/age/pom.xml | 14 ++++++
.../java/org/apache/hop/age/AgeGraphDatabase.java | 37 ++++++++++++++-
.../apache/hop/age/AgePostgreSqlDriverTest.java | 54 ++++++++++++++++++++++
14 files changed, 300 insertions(+), 22 deletions(-)
diff --git a/docker/integration-tests/integration-tests-mongo.yaml
b/docker/integration-tests/integration-tests-mongo.yaml
index 7b8f1c8e34..fb3d47df63 100644
--- a/docker/integration-tests/integration-tests-mongo.yaml
+++ b/docker/integration-tests/integration-tests-mongo.yaml
@@ -23,23 +23,31 @@ services:
depends_on:
mongo:
condition: service_healthy
+ mongo-atlas:
+ condition: service_healthy
links:
- mongo
+ - mongo-atlas
mongo-atlas:
image: mongodb/mongodb-atlas-local
- hostname: mongo
+ # The replica set member host is this hostname. It must be the Compose DNS
name clients use.
+ # "mongo" is the other service on this network, and a member named mongo
talks to mongo:8.
+ hostname: mongo-atlas
environment:
- MONGODB_INITDB_ROOT_USERNAME=root
- MONGODB_INITDB_ROOT_PASSWORD=example
ports:
- 27017:27017
+ # The image's own check waits until mongod has finished its init restart.
A plain ping
+ # succeeds during that restart, and the next SCRAM handshake then fails
with
+ # InterruptedAtShutdown.
healthcheck:
- test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"]
+ test: ["CMD", "runner", "healthcheck"]
interval: 10s
- timeout: 5s
- retries: 5
- start_period: 30s
+ timeout: 10s
+ retries: 12
+ start_period: 40s
mongo:
image: mongo:8
diff --git
a/engine/src/main/java/org/apache/hop/pipeline/engines/loadbalance/LoadBalancingPipelineEngine.java
b/engine/src/main/java/org/apache/hop/pipeline/engines/loadbalance/LoadBalancingPipelineEngine.java
index 3d00f0e777..2fc48e89b4 100644
---
a/engine/src/main/java/org/apache/hop/pipeline/engines/loadbalance/LoadBalancingPipelineEngine.java
+++
b/engine/src/main/java/org/apache/hop/pipeline/engines/loadbalance/LoadBalancingPipelineEngine.java
@@ -126,9 +126,12 @@ public class LoadBalancingPipelineEngine extends
RemotePipelineEngine {
assignment.setStatus(LoadBalancingAssignment.STATUS_RETRYING);
coordinator.saveAssignment(assignment);
}
- if (HopServerAdmission.isRetryableRegistrationFailure(e, containerId)
- && retry.canAttempt()) {
- logCapacityWait(attempt, e);
+ if (retry.canAttempt() && isRetryablePrepareFailure(e, containerId)) {
+ if (LoadBalancingCoordinator.isTransientProbeFailure(e)) {
+ logProbeWait(attempt, e);
+ } else {
+ logCapacityWait(attempt, e);
+ }
sleepBackoff(retry);
continue;
}
@@ -182,6 +185,20 @@ public class LoadBalancingPipelineEngine extends
RemotePipelineEngine {
return next;
}
+ private static boolean isRetryablePrepareFailure(Exception error, String
containerId) {
+ return HopServerAdmission.isRetryableRegistrationFailure(error,
containerId)
+ || (StringUtils.isEmpty(containerId)
+ && LoadBalancingCoordinator.isTransientProbeFailure(error));
+ }
+
+ private void logProbeWait(int attempt, Exception error) {
+ logChannel.logBasic(
+ "Load-balancing attempt "
+ + attempt
+ + " waiting for a Hop server to answer: "
+ + error.getMessage());
+ }
+
private void logCapacityWait(int attempt, Exception error) {
String server =
StringUtils.isEmpty(selectedHopServerName)
diff --git
a/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingCoordinator.java
b/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingCoordinator.java
index 0694c386a6..6b708637cd 100644
---
a/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingCoordinator.java
+++
b/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingCoordinator.java
@@ -220,7 +220,9 @@ public class LoadBalancingCoordinator<C> {
/**
* When every reachable server is full, throw {@link
HopServerAtCapacityException} so the engine
- * waits and retries. Unreachable or misconfigured groups stay a regular
{@link HopException}.
+ * waits and retries. A group that only timed out is also retried: a cold
Hop server often answers
+ * the status call a few seconds later. Misconfigured groups stay a regular
{@link HopException}
+ * and are not retried.
*/
static HopException noEligibleServerException(List<ServerHealthSnapshot>
snapshots) {
if (isGroupAtCapacity(snapshots)) {
@@ -272,6 +274,46 @@ public class LoadBalancingCoordinator<C> {
return null;
}
+ /**
+ * True when every reported server failed its status probe with a timeout or
a connection error. A
+ * missing server name or unknown Hop server metadata is a configuration
error and is not retried.
+ */
+ public static boolean isTransientProbeFailure(Throwable error) {
+ String message = probeFailureMessage(error);
+ if (message == null) {
+ return false;
+ }
+ boolean sawServer = false;
+ for (String line : message.split("\\R")) {
+ String trimmed = line.trim();
+ if (!trimmed.startsWith("- ")) {
+ continue;
+ }
+ sawServer = true;
+ if (!isTransientProbeReason(trimmed)) {
+ return false;
+ }
+ }
+ return sawServer;
+ }
+
+ private static String probeFailureMessage(Throwable error) {
+ for (Throwable current = error; current != null; current =
current.getCause()) {
+ String message = current.getMessage();
+ if (message != null && message.contains("No eligible Hop server")) {
+ return message;
+ }
+ if (current.getCause() == current) {
+ break;
+ }
+ }
+ return null;
+ }
+
+ private static boolean isTransientProbeReason(String line) {
+ return line.contains("timeout after ") || line.contains("Error querying
Hop server");
+ }
+
static String describeNoEligibleServer(List<ServerHealthSnapshot> snapshots)
{
StringBuilder message =
new StringBuilder("No eligible Hop server in the load-balancing
group:");
diff --git
a/engine/src/main/java/org/apache/hop/workflow/engines/loadbalance/LoadBalancingWorkflowEngine.java
b/engine/src/main/java/org/apache/hop/workflow/engines/loadbalance/LoadBalancingWorkflowEngine.java
index 032b398b0e..94eb7d89f7 100644
---
a/engine/src/main/java/org/apache/hop/workflow/engines/loadbalance/LoadBalancingWorkflowEngine.java
+++
b/engine/src/main/java/org/apache/hop/workflow/engines/loadbalance/LoadBalancingWorkflowEngine.java
@@ -112,9 +112,12 @@ public class LoadBalancingWorkflowEngine extends
RemoteWorkflowEngine {
assignment.setStatus(LoadBalancingAssignment.STATUS_RETRYING);
coordinator.saveAssignment(assignment);
}
- if (HopServerAdmission.isRetryableRegistrationFailure(e, containerId)
- && retry.canAttempt()) {
- logCapacityWait(attempt, e);
+ if (retry.canAttempt() && isRetryablePrepareFailure(e, containerId)) {
+ if (LoadBalancingCoordinator.isTransientProbeFailure(e)) {
+ logProbeWait(attempt, e);
+ } else {
+ logCapacityWait(attempt, e);
+ }
sleepBackoff(retry);
continue;
}
@@ -156,6 +159,20 @@ public class LoadBalancingWorkflowEngine extends
RemoteWorkflowEngine {
return next;
}
+ private static boolean isRetryablePrepareFailure(Exception error, String
containerId) {
+ return HopServerAdmission.isRetryableRegistrationFailure(error,
containerId)
+ || (StringUtils.isEmpty(containerId)
+ && LoadBalancingCoordinator.isTransientProbeFailure(error));
+ }
+
+ private void logProbeWait(int attempt, Exception error) {
+ logChannel.logBasic(
+ "Load-balancing attempt "
+ + attempt
+ + " waiting for a Hop server to answer: "
+ + error.getMessage());
+ }
+
private void logCapacityWait(int attempt, Exception error) {
String server =
StringUtils.isEmpty(selectedHopServerName)
diff --git
a/engine/src/test/java/org/apache/hop/server/loadbalance/LoadBalancingCoordinatorTest.java
b/engine/src/test/java/org/apache/hop/server/loadbalance/LoadBalancingCoordinatorTest.java
index b26fa2f7ad..ae595cd55b 100644
---
a/engine/src/test/java/org/apache/hop/server/loadbalance/LoadBalancingCoordinatorTest.java
+++
b/engine/src/test/java/org/apache/hop/server/loadbalance/LoadBalancingCoordinatorTest.java
@@ -66,6 +66,27 @@ class LoadBalancingCoordinatorTest {
assertTrue(error.getMessage().contains("No eligible Hop server"));
}
+ @Test
+ void probeTimeoutsAreRetried() {
+ HopException error =
+ LoadBalancingCoordinator.noEligibleServerException(
+ List.of(
+ ServerHealthSnapshot.unavailable("a", true, 2, "timeout after
3000ms"),
+ ServerHealthSnapshot.unavailable("b", true, 2, "timeout after
3000ms")));
+ assertFalse(error instanceof HopServerAtCapacityException);
+ assertTrue(LoadBalancingCoordinator.isTransientProbeFailure(error));
+ }
+
+ @Test
+ void unknownServerIsNotAProbeTimeout() {
+ HopException error =
+ LoadBalancingCoordinator.noEligibleServerException(
+ List.of(
+ ServerHealthSnapshot.unavailable("a", true, 2, "timeout after
3000ms"),
+ ServerHealthSnapshot.unavailable("b", true, 2, "Hop server 'b'
not found")));
+ assertFalse(LoadBalancingCoordinator.isTransientProbeFailure(error));
+ }
+
private static ServerHealthSnapshot eligible(String name, int occupying, int
max) {
ServerHealthSnapshot snapshot = new ServerHealthSnapshot();
snapshot.setHopServerName(name);
diff --git
a/integration-tests/load-balance/metadata/pipeline-run-configuration/load-balance.json
b/integration-tests/load-balance/metadata/pipeline-run-configuration/load-balance.json
index 4db3be0aaf..b793254bc7 100644
---
a/integration-tests/load-balance/metadata/pipeline-run-configuration/load-balance.json
+++
b/integration-tests/load-balance/metadata/pipeline-run-configuration/load-balance.json
@@ -6,7 +6,7 @@
"max_retries": "10",
"retry_window_ms": "30000",
"retry_on_execution_failure": false,
- "probe_timeout_ms": "3000",
+ "probe_timeout_ms": "15000",
"config_refresh_ms": "2000",
"state_folder": "${PROJECT_HOME}/state",
"server_poll_delay": "200",
diff --git
a/integration-tests/load-balance/metadata/workflow-run-configuration/load-balance.json
b/integration-tests/load-balance/metadata/workflow-run-configuration/load-balance.json
index 0cfe08588c..dce77b9874 100644
---
a/integration-tests/load-balance/metadata/workflow-run-configuration/load-balance.json
+++
b/integration-tests/load-balance/metadata/workflow-run-configuration/load-balance.json
@@ -6,7 +6,7 @@
"max_retries": "10",
"retry_window_ms": "30000",
"retry_on_execution_failure": false,
- "probe_timeout_ms": "3000",
+ "probe_timeout_ms": "15000",
"config_refresh_ms": "2000",
"state_folder": "${PROJECT_HOME}/state",
"server_poll_delay": "200",
diff --git a/integration-tests/samples/read-samples-build-hop-run.hpl
b/integration-tests/samples/read-samples-build-hop-run.hpl
index f04eb88576..f24c643a8a 100644
--- a/integration-tests/samples/read-samples-build-hop-run.hpl
+++ b/integration-tests/samples/read-samples-build-hop-run.hpl
@@ -583,6 +583,22 @@ var hop_run_cmd = './hop-run.sh -j samples -r ' +
run_config + ' -f ' + filename
<item>structured-extract-basic.hpl</item>
<item>requires an AI provider named ollama-local</item>
</line>
+ <line>
+ <item>extract-graph-basic.hpl</item>
+ <item>requires an AI provider named ollama-local</item>
+ </line>
+ <line>
+ <item>extract-graph-to-database.hpl</item>
+ <item>requires an AI provider named ollama-local and a graph
database</item>
+ </line>
+ <line>
+ <item>extract-graph-to-database.hwf</item>
+ <item>runs extract-graph-to-database.hpl</item>
+ </line>
+ <line>
+ <item>watch-files.hpl</item>
+ <item>watches a folder until it is stopped</item>
+ </line>
<line>
<item>call-db-procedure.hpl</item>
<item>requires a stored procedure in the database</item>
@@ -917,6 +933,36 @@ var hop_run_cmd = './hop-run.sh -j samples -r ' +
run_config + ' -f ' + filename
</value>
<conditions/>
</condition>
+ <condition>
+ <negated>Y</negated>
+ <operator>AND</operator>
+ <leftvalue>filename</leftvalue>
+ <function>CONTAINS</function>
+ <value>
+ <name>constant</name>
+ <type>String</type>
+ <text>/graph-databases/</text>
+ <length>-1</length>
+ <precision>-1</precision>
+ <isnull>N</isnull>
+ </value>
+ <conditions/>
+ </condition>
+ <condition>
+ <negated>Y</negated>
+ <operator>AND</operator>
+ <leftvalue>filename</leftvalue>
+ <function>CONTAINS</function>
+ <value>
+ <name>constant</name>
+ <type>String</type>
+ <text>/property-graph/</text>
+ <length>-1</length>
+ <precision>-1</precision>
+ <isnull>N</isnull>
+ </value>
+ <conditions/>
+ </condition>
<condition>
<negated>Y</negated>
<operator>AND</operator>
diff --git a/plugins/engines/beam/pom.xml b/plugins/engines/beam/pom.xml
index ef6f40d320..f9d660b599 100644
--- a/plugins/engines/beam/pom.xml
+++ b/plugins/engines/beam/pom.xml
@@ -307,11 +307,20 @@
</dependency>
<!-- DebeziumIO provides the CDC source but not a database connector.
Packaged with its PostgreSQL driver under
plugins/engines/beam/lib, not lib/core.
- A driver on the application classpath breaks PostgreSQL Bulk
Loader. -->
+ A driver on the application classpath breaks PostgreSQL Bulk
Loader.
+ The driver is a direct dependency on purpose: the Beam assembly
exclude is not
+ applied to transitive jars, so a driver pulled only through
Debezium still lands
+ in lib/core. -->
<dependency>
<groupId>io.debezium</groupId>
<artifactId>debezium-connector-postgres</artifactId>
<version>3.1.3.Final</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
@@ -1656,6 +1665,13 @@
</exclusion>
</exclusions>
</dependency>
+ <!-- Direct, not a Debezium transitive: the Beam assembly exclude does
not apply to
+ transitive jars, and this driver must stay out of lib/core. -->
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>42.7.12</version>
+ </dependency>
<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-transform-addsequence</artifactId>
diff --git a/plugins/engines/beam/src/assembly/assembly.xml
b/plugins/engines/beam/src/assembly/assembly.xml
index e8d0af5f38..c088ad5156 100644
--- a/plugins/engines/beam/src/assembly/assembly.xml
+++ b/plugins/engines/beam/src/assembly/assembly.xml
@@ -91,11 +91,13 @@
Snowflake IO jar that loads it, off the shared lib/core
classpath. -->
<exclude>org.apache.beam:beam-sdks-java-io-snowflake</exclude>
<exclude>net.snowflake:snowflake-jdbc</exclude>
- <!-- debezium-connector-postgres pulls
org.postgresql:postgresql. A copy of that
- driver on lib/core is loaded by the application
classloader, so DriverManager
- returns a PgConnection the database plugin cannot cast to
its own
- PGConnection (PostgreSQL Bulk Loader). Keep the connector
and its driver in
- the Beam plugin classloader. -->
+ <!-- debezium-connector-postgres used to pull
org.postgresql:postgresql
+ transitively. Assembly excludes are not applied to
transitive jars, so that
+ copy stayed in lib/core and DriverManager returned an
app-classloader
+ PgConnection the database plugin cannot cast to its own
PGConnection
+ (PostgreSQL Bulk Loader). The driver is now a direct
dependency and this
+ exclude keeps it out of lib/core. The connector and the
driver are packaged
+ under plugins/engines/beam/lib. -->
<exclude>io.debezium:debezium-connector-postgres</exclude>
<exclude>org.postgresql:postgresql</exclude>
</excludes>
diff --git
a/plugins/misc/marketplace/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml
b/plugins/misc/marketplace/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml
index 653c237c38..b6d10c1205 100644
---
a/plugins/misc/marketplace/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml
+++
b/plugins/misc/marketplace/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml
@@ -247,6 +247,14 @@ plugins:
description: HDFS via WebHDFS, HttpFS or Knox (Kerberos). No Hadoop client
libraries.
installPath: plugins/tech/hadoop
modulePath: plugins/tech/hadoop
+ - artifactId: hop-tech-lakehouse
+ name: Lakehouse
+ category: Technology
+ description: >-
+ Lake table input and output for Apache Iceberg. The native Spark engine
loads these
+ transform classes from this plugin, so install it together with the
Spark engine.
+ installPath: plugins/tech/lakehouse
+ modulePath: plugins/tech/lakehouse
- artifactId: hop-tech-databricks
name: Databricks
category: Technology
diff --git a/plugins/tech/age/pom.xml b/plugins/tech/age/pom.xml
index 096fa34b41..771f0a93d7 100644
--- a/plugins/tech/age/pom.xml
+++ b/plugins/tech/age/pom.xml
@@ -29,6 +29,11 @@
<name>Hop Plugins Technology Apache AGE</name>
<properties>
+ <!-- The driver is provided so AGE compiles against JDBC. It already
ships in lib/jdbc
+ with the PostgreSQL database plugin. Copying this provided jar
into lib/core makes
+ DriverManager return an app-classloader PgConnection that
PostgreSQL Bulk Loader
+ cannot cast to the database plugin's PGConnection. -->
+ <hop.libcore.exclude1>org.postgresql:postgresql</hop.libcore.exclude1>
<testcontainers.version>1.21.4</testcontainers.version>
</properties>
@@ -51,6 +56,15 @@
<artifactId>postgresql</artifactId>
<scope>provided</scope>
</dependency>
+ <!-- AgeGraphDatabase loads the JDBC driver from this plugin. The
assembled client
+ already has it. These tests need it on the classpath, and test
scope keeps the
+ plugin jar out of lib/core. -->
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-databases-postgresql</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-tech-neo4j</artifactId>
diff --git
a/plugins/tech/age/src/main/java/org/apache/hop/age/AgeGraphDatabase.java
b/plugins/tech/age/src/main/java/org/apache/hop/age/AgeGraphDatabase.java
index 7dc94f3b4e..60138b14ed 100644
--- a/plugins/tech/age/src/main/java/org/apache/hop/age/AgeGraphDatabase.java
+++ b/plugins/tech/age/src/main/java/org/apache/hop/age/AgeGraphDatabase.java
@@ -18,7 +18,7 @@
package org.apache.hop.age;
import java.sql.Connection;
-import java.sql.DriverManager;
+import java.sql.Driver;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -31,6 +31,7 @@ import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
import org.apache.hop.core.Const;
+import org.apache.hop.core.database.DatabasePluginType;
import org.apache.hop.core.encryption.Encr;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.graph.BaseGraphDatabase;
@@ -43,6 +44,8 @@ import org.apache.hop.core.gui.plugin.GuiPlugin;
import org.apache.hop.core.gui.plugin.GuiWidgetElement;
import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.plugins.IPlugin;
+import org.apache.hop.core.plugins.PluginRegistry;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.metadata.api.HopMetadataProperty;
@@ -60,6 +63,8 @@ public class AgeGraphDatabase extends BaseGraphDatabase {
private static final Map<String, Object> GRAPH_CREATION_LOCKS = new
ConcurrentHashMap<>();
private static final String GROUP =
"i18n::AgeGraphDatabase.Group.Connection";
+ private static final String POSTGRESQL_PLUGIN_ID = "POSTGRESQL";
+ private static final String POSTGRESQL_DRIVER_CLASS =
"org.postgresql.Driver";
@GuiWidgetElement(
id = "hostname",
@@ -190,11 +195,15 @@ public class AgeGraphDatabase extends BaseGraphDatabase {
}
Connection connection;
try {
- connection = DriverManager.getConnection(getUrl(variables), properties);
+ connection = postgresqlDriver().connect(getUrl(variables), properties);
} catch (SQLException e) {
throw new HopException(
"Unable to connect to " + getUrl(variables) + " for " +
connectionName, e);
}
+ if (connection == null) {
+ throw new HopException(
+ "The PostgreSQL driver does not accept " + getUrl(variables) + " for
" + connectionName);
+ }
try {
prepareSession(log, connection, realGraphName, getUrl(variables) + "|" +
realGraphName);
return new AgeGraphConnection(connection, realGraphName, log);
@@ -208,6 +217,30 @@ public class AgeGraphDatabase extends BaseGraphDatabase {
}
}
+ /**
+ * The PostgreSQL driver from the PostgreSQL database plugin. That plugin's
classloader holds the
+ * shared JDBC folders (lib/jdbc, or HOP_SHARED_JDBC_FOLDERS), which a graph
database plugin does
+ * not see. A driver on lib/core would break PostgreSQL Bulk Loader.
+ */
+ private static Driver postgresqlDriver() throws HopException {
+ PluginRegistry registry = PluginRegistry.getInstance();
+ IPlugin plugin = registry.getPlugin(DatabasePluginType.class,
POSTGRESQL_PLUGIN_ID);
+ if (plugin == null) {
+ throw new HopException("Apache AGE needs the PostgreSQL database
plugin");
+ }
+ try {
+ return (Driver)
+ registry
+ .getClassLoader(plugin)
+ .loadClass(POSTGRESQL_DRIVER_CLASS)
+ .getDeclaredConstructor()
+ .newInstance();
+ } catch (Exception | LinkageError e) {
+ throw new HopException(
+ "Unable to load " + POSTGRESQL_DRIVER_CLASS + " from the PostgreSQL
database plugin", e);
+ }
+ }
+
/** Load AGE, put its catalog on the search path and create the graph if
needed. */
private void prepareSession(
ILogChannel log, Connection connection, String realGraphName, String
lockKey)
diff --git
a/plugins/tech/age/src/test/java/org/apache/hop/age/AgePostgreSqlDriverTest.java
b/plugins/tech/age/src/test/java/org/apache/hop/age/AgePostgreSqlDriverTest.java
new file mode 100644
index 0000000000..0433ceb8e9
--- /dev/null
+++
b/plugins/tech/age/src/test/java/org/apache/hop/age/AgePostgreSqlDriverTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.age;
+
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.sql.SQLException;
+import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.variables.Variables;
+import org.junit.jupiter.api.Test;
+
+/** The driver is taken from the PostgreSQL database plugin, which these tests
depend on. */
+class AgePostgreSqlDriverTest {
+
+ @Test
+ void theDriverLoadsFromThePostgreSqlPlugin() throws Exception {
+ HopClientEnvironment.init();
+ AgeGraphDatabase database = new AgeGraphDatabase();
+ database.setHostname("127.0.0.1");
+ database.setPort("1");
+ database.setDatabaseName("hop");
+ database.setUsername("hop");
+ database.setPassword("hop");
+ database.setGraphName("hop_it");
+
+ HopException exception =
+ assertThrows(
+ HopException.class,
+ () ->
+ database.connect(
+ LogChannel.GENERAL, Variables.getADefaultVariableSpace(),
"driver"));
+
+ // The driver accepted the URL. Nothing is listening on that port.
+ assertInstanceOf(SQLException.class, exception.getCause(),
exception.getMessage());
+ }
+}
