This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 267be5906f Add some extra logging and timeout options, fixes #7452
(#7454)
267be5906f is described below
commit 267be5906fcf4630cc3f36067892fd26b925a26e
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Wed Jul 8 14:30:25 2026 +0200
Add some extra logging and timeout options, fixes #7452 (#7454)
---
core/src/main/java/org/apache/hop/core/Const.java | 30 ++++
.../org/apache/hop/core/database/Database.java | 65 ++++++++
.../hop/core/database/DatabaseTimeoutTest.java | 175 +++++++++++++++++++++
.../modules/ROOT/pages/database/databases.adoc | 20 +++
.../modules/ROOT/pages/variables.adoc | 8 +
5 files changed, 298 insertions(+)
diff --git a/core/src/main/java/org/apache/hop/core/Const.java
b/core/src/main/java/org/apache/hop/core/Const.java
index 9ab20b8003..a1d5db35df 100644
--- a/core/src/main/java/org/apache/hop/core/Const.java
+++ b/core/src/main/java/org/apache/hop/core/Const.java
@@ -176,6 +176,36 @@ public class Const {
description = "A comma separated list pointing to folders with JDBC
drivers to add.")
public static final String HOP_SHARED_JDBC_FOLDERS =
"HOP_SHARED_JDBC_FOLDERS";
+ /**
+ * Default login/connection timeout (in seconds) applied when opening a JDBC
database connection.
+ * Maps to {@link java.sql.DriverManager#setLoginTimeout(int)} and bounds
how long Hop waits while
+ * establishing a connection. Defaults to 30 seconds; set to 0 to impose no
timeout (a stalled
+ * connect can then block indefinitely). Note the setting is advisory:
individual JDBC drivers may
+ * ignore it. It can still be overridden per connection via driver-specific
connection options.
+ */
+ @Variable(
+ value = "30",
+ description =
+ "Default login/connection timeout in seconds when opening a JDBC
database connection (DriverManager.setLoginTimeout). Defaults to 30; set to 0
to impose no timeout. Advisory: some JDBC drivers ignore it.")
+ public static final String HOP_DATABASE_CONNECTION_TIMEOUT =
"HOP_DATABASE_CONNECTION_TIMEOUT";
+
+ /**
+ * Default network/socket timeout (in seconds) applied to a JDBC database
connection after it has
+ * been opened. Maps to {@link
+ * java.sql.Connection#setNetworkTimeout(java.util.concurrent.Executor,
int)} and bounds how long
+ * a driver waits for a database request to complete. A value of 0 (default)
means no timeout is
+ * imposed. WARNING: when set, this applies to every request on the
connection, so any single
+ * query, read or bulk operation that runs longer than the timeout is
aborted and the connection
+ * is closed - use a value large enough for your longest-running SQL.
Drivers that do not support
+ * the operation are ignored (logged at detailed level). This guards reads
on an established
+ * connection; the initial handshake read still relies on the driver's own
socket-timeout option.
+ */
+ @Variable(
+ value = "0",
+ description =
+ "Default network/socket timeout in seconds applied to a JDBC
connection after it is opened (Connection.setNetworkTimeout). 0 (default) means
no timeout is imposed. WARNING: when set, any single query/read running longer
than this is aborted - use a value large enough for your longest-running SQL.
Unsupported drivers are ignored.")
+ public static final String HOP_DATABASE_SOCKET_TIMEOUT =
"HOP_DATABASE_SOCKET_TIMEOUT";
+
/**
* Per-run override for the engine-compatibility gate in {@code
Pipeline.prepareExecution} /
* {@code Workflow.startExecution}. Set to 'Y' to run a pipeline or workflow
that contains
diff --git a/core/src/main/java/org/apache/hop/core/database/Database.java
b/core/src/main/java/org/apache/hop/core/database/Database.java
index 73001b73fc..0eaf15c009 100644
--- a/core/src/main/java/org/apache/hop/core/database/Database.java
+++ b/core/src/main/java/org/apache/hop/core/database/Database.java
@@ -430,6 +430,10 @@ public class Database implements IVariables,
ILoggingObject, AutoCloseable {
PluginRegistry.getInstance()
.getPlugin(DatabasePluginType.class, databaseMeta.getIDatabase());
+ if (log.isDetailed()) {
+ log.logDetailed("Loading JDBC driver class '" + classname + "'");
+ }
+
try {
synchronized (DriverManager.class) {
ClassLoader classLoader =
PluginRegistry.getInstance().getClassLoader(plugin);
@@ -495,6 +499,14 @@ public class Database implements IVariables,
ILoggingObject, AutoCloseable {
Properties properties = databaseMeta.getConnectionProperties(this);
+ int connectionTimeout = applyLoginTimeout();
+
+ if (log.isDetailed()) {
+ log.logDetailed(
+ "Opening JDBC connection..."
+ + (connectionTimeout > 0 ? " (login timeout " +
connectionTimeout + "s)" : ""));
+ }
+
if (databaseMeta.supportsOptionsInURL()) {
if (!Utils.isEmpty(username) || !Utils.isEmpty(password)) {
// Allow for empty username with given password, in this case
username must be given with
@@ -525,12 +537,65 @@ public class Database implements IVariables,
ILoggingObject, AutoCloseable {
connection = DriverManager.getConnection(url, properties);
}
+
+ if (log.isDetailed()) {
+ log.logDetailed("JDBC connection opened successfully");
+ }
+
+ applyNetworkTimeout();
} catch (Exception e) {
throw new HopDatabaseException(
"Error connecting to database: (using class " + classname + ")", e);
}
}
+ /**
+ * Apply the default login/connection timeout ({@link
Const#HOP_DATABASE_CONNECTION_TIMEOUT}, in
+ * seconds) via {@link DriverManager#setLoginTimeout(int)}. This is a
JVM-wide setting that bounds
+ * how long the driver waits while establishing a connection; the default is
30 seconds and a
+ * value of 0 leaves the current default in place. The setting is advisory -
some JDBC drivers
+ * ignore it in favor of their own connection options.
+ *
+ * @return the resolved connection timeout in seconds
+ */
+ int applyLoginTimeout() {
+ int connectionTimeout =
Const.toInt(getVariable(Const.HOP_DATABASE_CONNECTION_TIMEOUT), 30);
+ if (connectionTimeout > 0) {
+ DriverManager.setLoginTimeout(connectionTimeout);
+ }
+ return connectionTimeout;
+ }
+
+ /**
+ * Apply the default network/socket timeout ({@link
Const#HOP_DATABASE_SOCKET_TIMEOUT}, in
+ * seconds) to the freshly opened connection via {@link
+ * Connection#setNetworkTimeout(java.util.concurrent.Executor, int)}. This
bounds reads on an
+ * already-established connection. A value of 0 (default) leaves it
unbounded. Drivers that do not
+ * support the operation are ignored - an optional timeout must never fail
the connection.
+ */
+ void applyNetworkTimeout() {
+ int socketTimeout =
Const.toInt(getVariable(Const.HOP_DATABASE_SOCKET_TIMEOUT), 0);
+ if (socketTimeout <= 0 || connection == null) {
+ return;
+ }
+ try {
+ // Execute the driver's timeout callback inline; no extra thread pool to
leak.
+ connection.setNetworkTimeout(command -> command.run(), socketTimeout *
1000);
+ if (log.isDetailed()) {
+ log.logDetailed(
+ "Applied network/socket timeout of " + socketTimeout + "s to the
connection");
+ }
+ } catch (Exception e) {
+ if (log.isDetailed()) {
+ log.logDetailed(
+ "Unable to apply network/socket timeout ("
+ + socketTimeout
+ + "s); driver may not support it: "
+ + e.getMessage());
+ }
+ }
+ }
+
/** close() and disconnect() are the same. */
@Override
public synchronized void close() {
diff --git
a/core/src/test/java/org/apache/hop/core/database/DatabaseTimeoutTest.java
b/core/src/test/java/org/apache/hop/core/database/DatabaseTimeoutTest.java
new file mode 100644
index 0000000000..d7a6666a1c
--- /dev/null
+++ b/core/src/test/java/org/apache/hop/core/database/DatabaseTimeoutTest.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.database;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLFeatureNotSupportedException;
+import java.util.concurrent.Executor;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.junit.rules.RestoreHopEnvironmentExtension;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/**
+ * Verifies the login/connection ({@link
Const#HOP_DATABASE_CONNECTION_TIMEOUT}) and network/socket
+ * ({@link Const#HOP_DATABASE_SOCKET_TIMEOUT}) timeout defaults are resolved
and applied correctly.
+ */
+@ExtendWith(RestoreHopEnvironmentExtension.class)
+class DatabaseTimeoutTest {
+
+ private int originalLoginTimeout;
+
+ @BeforeAll
+ static void setUpClass() throws Exception {
+ HopClientEnvironment.init();
+ }
+
+ @BeforeEach
+ void rememberLoginTimeout() {
+ // DriverManager.setLoginTimeout is a JVM-wide setting; snapshot it so
tests don't leak state.
+ originalLoginTimeout = DriverManager.getLoginTimeout();
+ }
+
+ @AfterEach
+ void restoreLoginTimeout() {
+ DriverManager.setLoginTimeout(originalLoginTimeout);
+ }
+
+ private Database databaseWith(String variableName, String value) {
+ Variables variables = new Variables();
+ if (value != null) {
+ variables.setVariable(variableName, value);
+ }
+ DatabaseMeta meta = new DatabaseMeta("test", "None", "", "", "", "", "",
"");
+ return new Database(null, variables, meta);
+ }
+
+ //
---------------------------------------------------------------------------
+ // Login / connection timeout -> DriverManager.setLoginTimeout(seconds)
+ //
---------------------------------------------------------------------------
+
+ @Test
+ void loginTimeout_defaultsTo30SecondsWhenUnset() {
+ DriverManager.setLoginTimeout(0);
+ Database db = databaseWith(Const.HOP_DATABASE_CONNECTION_TIMEOUT, null);
+
+ int resolved = db.applyLoginTimeout();
+
+ assertEquals(30, resolved);
+ assertEquals(30, DriverManager.getLoginTimeout());
+ }
+
+ @Test
+ void loginTimeout_appliesConfiguredValue() {
+ DriverManager.setLoginTimeout(0);
+ Database db = databaseWith(Const.HOP_DATABASE_CONNECTION_TIMEOUT, "45");
+
+ int resolved = db.applyLoginTimeout();
+
+ assertEquals(45, resolved);
+ assertEquals(45, DriverManager.getLoginTimeout());
+ }
+
+ @Test
+ void loginTimeout_zeroLeavesDriverManagerUntouched() {
+ DriverManager.setLoginTimeout(7);
+ Database db = databaseWith(Const.HOP_DATABASE_CONNECTION_TIMEOUT, "0");
+
+ int resolved = db.applyLoginTimeout();
+
+ // 0 means "impose no timeout" - the existing JVM default must be left in
place.
+ assertEquals(0, resolved);
+ assertEquals(7, DriverManager.getLoginTimeout());
+ }
+
+ //
---------------------------------------------------------------------------
+ // Network / socket timeout -> Connection.setNetworkTimeout(executor, millis)
+ //
---------------------------------------------------------------------------
+
+ @Test
+ void socketTimeout_isDisabledByDefault() throws Exception {
+ Connection connection = mock(Connection.class);
+ Database db = databaseWith(Const.HOP_DATABASE_SOCKET_TIMEOUT, null);
+ db.setConnection(connection);
+
+ db.applyNetworkTimeout();
+
+ verify(connection, never()).setNetworkTimeout(any(), anyInt());
+ }
+
+ @Test
+ void socketTimeout_zeroIsDisabled() throws Exception {
+ Connection connection = mock(Connection.class);
+ Database db = databaseWith(Const.HOP_DATABASE_SOCKET_TIMEOUT, "0");
+ db.setConnection(connection);
+
+ db.applyNetworkTimeout();
+
+ verify(connection, never()).setNetworkTimeout(any(), anyInt());
+ }
+
+ @Test
+ void socketTimeout_appliesConfiguredValueConvertedToMillis() throws
Exception {
+ Connection connection = mock(Connection.class);
+ Database db = databaseWith(Const.HOP_DATABASE_SOCKET_TIMEOUT, "30");
+ db.setConnection(connection);
+
+ db.applyNetworkTimeout();
+
+ // seconds -> milliseconds
+ verify(connection, times(1)).setNetworkTimeout(any(Executor.class),
eq(30000));
+ }
+
+ @Test
+ void socketTimeout_nullConnectionIsNoop() {
+ Database db = databaseWith(Const.HOP_DATABASE_SOCKET_TIMEOUT, "30");
+ // No setConnection() call - the connection stays null.
+
+ assertDoesNotThrow(db::applyNetworkTimeout);
+ }
+
+ @Test
+ void socketTimeout_unsupportedDriverIsIgnored() throws Exception {
+ Connection connection = mock(Connection.class);
+ doThrow(new SQLFeatureNotSupportedException("not supported"))
+ .when(connection)
+ .setNetworkTimeout(any(), anyInt());
+ Database db = databaseWith(Const.HOP_DATABASE_SOCKET_TIMEOUT, "30");
+ db.setConnection(connection);
+
+ // An optional timeout must never fail the connection.
+ assertDoesNotThrow(db::applyNetworkTimeout);
+ }
+}
diff --git a/docs/hop-user-manual/modules/ROOT/pages/database/databases.adoc
b/docs/hop-user-manual/modules/ROOT/pages/database/databases.adoc
index eae9f201ee..4d3f0dfc71 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/database/databases.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/database/databases.adoc
@@ -112,6 +112,26 @@ jdbc:sqlserver://localhost:1433;" +
|trustServerCertificate|true
|===
+== Connection and socket timeouts
+
+If a database host becomes unreachable at the wrong moment (for example a
half-open TCP connection that is never answered), a transform can block
indefinitely while it waits for the driver. This makes a pipeline appear to
"hang" during initialization, even though the databases themselves are healthy.
+
+Two variables set a default timeout for *every* JDBC connection Hop opens:
+
+[%header, width="90%", cols="2,1,4"]
+|===
+|Variable|Default|Description
+|`HOP_DATABASE_CONNECTION_TIMEOUT`|`30`|Login/connection timeout in
**seconds** applied while establishing a connection (maps to
`DriverManager.setLoginTimeout`). Set to `0` to leave the connect unbounded.
The setting is advisory - some drivers ignore it in favor of their own
connection options.
+|`HOP_DATABASE_SOCKET_TIMEOUT`|`0`|Network/socket timeout in **seconds**
applied to a connection once it is open (maps to
`Connection.setNetworkTimeout`). `0` (default) leaves reads unbounded.
Unsupported drivers are ignored.
+|===
+
+[WARNING]
+====
+`HOP_DATABASE_SOCKET_TIMEOUT` is disabled by default (`0`) on purpose: when
set, it applies to *every* request on the connection, not just the initial
read. Any single query, read or bulk operation that runs longer than the
timeout is aborted and its connection is closed. If your pipelines contain
long-running SQL (heavy aggregations, large bulk loads, slow reports), only
enable it with a value large enough to cover them.
+====
+
+These are global defaults applied to all connections. For finer control - or
to bound the *initial* handshake read, which happens inside the driver before
Hop can apply the socket timeout - set the driver's own timeout options in the
connection's `Options` tab (see the Options table above). For example:
+
== Hard-coded JDBC to Hop data type mappings
Below are the mappings applied by Hop when converting from database column
types to Hop data types when `HOP_DB_DDL_COMPATIBLE` is `false` (the default).
diff --git a/docs/hop-user-manual/modules/ROOT/pages/variables.adoc
b/docs/hop-user-manual/modules/ROOT/pages/variables.adoc
index fd767e1096..763424ee72 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/variables.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/variables.adoc
@@ -250,6 +250,14 @@ Otherwise by default NULL is ignored by the MIN aggregate
and MIN is set to the
See also the variable HOP_AGGREGATION_ALL_NULLS_ARE_ZERO.
|HOP_ALLOW_EMPTY_FIELD_NAMES_AND_TYPES|N|Set this variable to Y to allow your
pipeline to pass 'null' fields and/or empty types.
|HOP_BATCHING_ROWSET|N|Set this variable to 'Y' if you want to test a more
efficient batching row set.
+|HOP_DATABASE_CONNECTION_TIMEOUT|30|Default login/connection timeout in
**seconds** applied when opening a JDBC database connection (maps to
`DriverManager.setLoginTimeout`).
+Defaults to `30`; set to `0` to impose no timeout (a stalled connect can then
block indefinitely).
+The setting is advisory: some JDBC drivers ignore it in favor of their own
connection options.
+|HOP_DATABASE_SOCKET_TIMEOUT|0|Default network/socket timeout in **seconds**
applied to a JDBC database connection after it is opened (maps to
`Connection.setNetworkTimeout`).
+`0` (default) means no timeout is imposed.
+WARNING: when set, this applies to *every* request on the connection, so any
single query, read or bulk operation that runs longer than the timeout is
aborted and the connection is closed. Use a value large enough for your
longest-running SQL.
+This bounds reads on an already-established connection; the initial handshake
read still relies on the driver's own socket-timeout connection option.
+Drivers that do not support the operation are ignored.
|HOP_DEFAULT_BIGNUMBER_FORMAT||The name of the variable containing an
alternative default bignumber format
|HOP_DEFAULT_BUFFER_POLLING_WAITTIME|20|This is the default polling frequency
for the transforms input buffer (in ms)
|HOP_DEFAULT_DATE_FORMAT||The name of the variable containing an alternative
default date format
