This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 8111830d79 Issue #8657 : Force refresh the Problems tab after clearing
the database cache (#8940)
8111830d79 is described below
commit 8111830d790727d7ade538e69450457d6eacd0f2
Author: Matt Casters <[email protected]>
AuthorDate: Mon Oct 19 09:32:35 2026 +0200
Issue #8657 : Force refresh the Problems tab after clearing the database
cache (#8940)
* Issue #8657 : Force refresh the Problems tab after clearing the database
cache
The Problems remarks kept the columns from the first check. Force refresh
clears the database cache and the open pipelines' field caches, checks the
active editor, and re-lints the others. A query or lint pass that started
before the clear does not write its old result back.
* Issue #8657 : Keep lint results across unrelated database cache clears
A Problems-tab force refresh bumps its own counter and drops only the passes
that started before that click. Clearing the cache from the broom, the SQL
editor, or a DDL statement no longer throws a running lint away. The active
editor is re-linted as well, because its check does not always republish
markers.
---
.../src/main/java/org/apache/hop/core/DbCache.java | 65 +++++++---
.../org/apache/hop/core/database/Database.java | 9 +-
.../hop/core/extension/HopExtensionPoint.java | 8 ++
.../test/java/org/apache/hop/core/DbCacheTest.java | 25 ++++
.../org/apache/hop/core/database/DatabaseTest.java | 47 +++++++
.../org/apache/hop/lint/BackgroundLintService.java | 122 +++++++++++++++--
.../org/apache/hop/lint/LintEditorGraphHelper.java | 25 ++++
.../lint/ProblemsForceRefreshLintExtension.java | 46 +++++++
.../hop/lint/ProblemsForceRefreshLintTest.java | 144 +++++++++++++++++++++
.../delegates/HopGuiPipelineCheckDelegate.java | 15 +++
.../hopgui/file/shared/ProblemsForceRefresh.java | 115 ++++++++++++++++
.../delegates/HopGuiWorkflowCheckDelegate.java | 15 +++
.../ui/hopgui/messages/messages_en_US.properties | 1 +
.../file/shared/ProblemsForceRefreshTest.java | 72 +++++++++++
14 files changed, 684 insertions(+), 25 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/DbCache.java
b/core/src/main/java/org/apache/hop/core/DbCache.java
index 3153667ce9..c04337da32 100644
--- a/core/src/main/java/org/apache/hop/core/DbCache.java
+++ b/core/src/main/java/org/apache/hop/core/DbCache.java
@@ -40,15 +40,46 @@ public class DbCache {
*/
private final AtomicInteger generation = new AtomicInteger();
+ /** Held for a clear and for a put that has to agree with that clear. */
+ private final Object lock = new Object();
+
@Getter @Setter private boolean active;
public void put(DbCacheEntry entry, IRowMeta fields) {
- if (!active) {
+ if (!active || entry == null || fields == null) {
return;
}
- IRowMeta copy = fields.clone();
- cache.put(entry, copy);
+ synchronized (lock) {
+ cache.put(entry, fields.clone());
+ }
+ }
+
+ /**
+ * Store a row that was just read from the database, unless the cache was
cleared after that read
+ * started.
+ *
+ * <p>A check remembers {@link #getGeneration()} before it queries. Clearing
the cache while the
+ * query is in flight bumps the generation, and the row from that query is
the layout from before
+ * the clear. Storing it would put the old columns straight back.
+ *
+ * @param entry the query that produced the row
+ * @param fields the row read from the database
+ * @param generationAtQuery the generation observed before the query
+ * @return true when the row was stored
+ */
+ public boolean putIfCurrent(DbCacheEntry entry, IRowMeta fields, int
generationAtQuery) {
+ if (!active || entry == null || fields == null) {
+ return false;
+ }
+
+ synchronized (lock) {
+ if (generation.get() != generationAtQuery) {
+ return false;
+ }
+ cache.put(entry, fields.clone());
+ return true;
+ }
}
/**
@@ -81,21 +112,25 @@ public class DbCache {
* to clear it all.
*/
public void clear(String dbname) {
- if (dbname == null) {
- cache = new Hashtable<>();
- } else {
- Enumeration<DbCacheEntry> keys = cache.keys();
- while (keys.hasMoreElements()) {
- DbCacheEntry entry = keys.nextElement();
- if (entry.sameDB(dbname)) {
- // Same name: remove it!
- cache.remove(entry);
+ synchronized (lock) {
+ if (dbname == null) {
+ cache = new Hashtable<>();
+ } else {
+ Enumeration<DbCacheEntry> keys = cache.keys();
+ while (keys.hasMoreElements()) {
+ DbCacheEntry entry = keys.nextElement();
+ if (entry.sameDB(dbname)) {
+ // Same name: remove it!
+ cache.remove(entry);
+ }
}
}
+ // Only bump the generation once the entries are gone. A reader which
sees the new generation
+ // and re-derives its row metadata must not be able to pick up the
entries we are removing.
+ // putIfCurrent holds the same lock, so a query that started before this
clear cannot store
+ // its row after the generation moves.
+ generation.incrementAndGet();
}
- // Only bump the generation once the entries are gone. A reader which sees
the new generation
- // and re-derives its row metadata must not be able to pick up the entries
we are removing.
- generation.incrementAndGet();
}
/**
diff --git a/core/src/main/java/org/apache/hop/core/database/Database.java
b/core/src/main/java/org/apache/hop/core/database/Database.java
index 4ba3b61483..4fa7b33ded 100644
--- a/core/src/main/java/org/apache/hop/core/database/Database.java
+++ b/core/src/main/java/org/apache/hop/core/database/Database.java
@@ -2305,6 +2305,11 @@ public class Database implements IVariables,
ILoggingObject, AutoCloseable {
return null; // Cache test without connect.
}
+ // Remember the generation before asking the database. A clear that lands
while the query is in
+ // flight has to discard this row: it describes the layout from before the
clear, and storing
+ // it would put those columns straight back. See DbCache.putIfCurrent.
+ int generationAtQuery = dbcache == null ? 0 : dbcache.getGeneration();
+
// No cache entry found
// The new method of retrieving the query fields fails on Oracle because
@@ -2340,9 +2345,9 @@ public class Database implements IVariables,
ILoggingObject, AutoCloseable {
}
}
- // Store in cache!!
+ // Store in cache when this read still belongs to the generation it
started in.
if (dbcache != null && fields != null) {
- dbcache.put(entry, fields);
+ dbcache.putIfCurrent(entry, fields, generationAtQuery);
}
return fields;
diff --git
a/core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java
b/core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java
index 45329d9134..8ea060b23e 100644
--- a/core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java
+++ b/core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java
@@ -109,6 +109,14 @@ public enum HopExtensionPoint {
AfterCheckTransform("After a transform has been checked for
warnings/errors."),
AfterCheckActions("After a set of workflow actions has been checked for
warnings/errors."),
+ /**
+ * The Problems tab force-refresh button has cleared the database cache. The
payload is the active
+ * editor ({@code HopGuiAbstractGraph}). Listeners re-lint every open
editor, including that one.
+ * The button also re-checks the active editor itself.
+ */
+ ProblemsForceRefresh(
+ "Problems tab force refresh, after the database cache was cleared
(active HopGuiAbstractGraph)"),
+
HopServerInit("Right before the Hop server starts"),
HopServerStartup("Right after the Hop server has started and is fully
functional"),
HopServerShutdown("Right before the Hop server will shutdown"),
diff --git a/core/src/test/java/org/apache/hop/core/DbCacheTest.java
b/core/src/test/java/org/apache/hop/core/DbCacheTest.java
index f19a0aea24..56a0031688 100644
--- a/core/src/test/java/org/apache/hop/core/DbCacheTest.java
+++ b/core/src/test/java/org/apache/hop/core/DbCacheTest.java
@@ -17,6 +17,7 @@
package org.apache.hop.core;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -50,6 +51,30 @@ class DbCacheTest {
assertNotNull(cache.get(new DbCacheEntry("two", "select * from t")));
}
+ @Test
+ void aRowQueriedAcrossAClearIsNotStored() {
+ DbCache cache = DbCache.getInstance();
+ cache.clear(null);
+ int generation = cache.getGeneration();
+ cache.clear(null);
+
+ assertFalse(
+ cache.putIfCurrent(new DbCacheEntry("one", "select * from t"),
rowMeta(), generation));
+ assertNull(cache.get(new DbCacheEntry("one", "select * from t")));
+ }
+
+ @Test
+ void aRowQueriedAfterAClearIsStored() {
+ DbCache cache = DbCache.getInstance();
+ cache.clear(null);
+ int generation = cache.getGeneration();
+
+ assertTrue(
+ cache.putIfCurrent(new DbCacheEntry("one", "select * from t"),
rowMeta(), generation));
+ assertEquals(
+ "a", cache.get(new DbCacheEntry("one", "select * from
t")).getValueMeta(0).getName());
+ }
+
@Test
void clearingBumpsTheGeneration() {
DbCache cache = DbCache.getInstance();
diff --git a/core/src/test/java/org/apache/hop/core/database/DatabaseTest.java
b/core/src/test/java/org/apache/hop/core/database/DatabaseTest.java
index 637d9524ca..ad23986036 100644
--- a/core/src/test/java/org/apache/hop/core/database/DatabaseTest.java
+++ b/core/src/test/java/org/apache/hop/core/database/DatabaseTest.java
@@ -31,6 +31,7 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.nullable;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
@@ -54,6 +55,8 @@ import java.sql.Statement;
import java.sql.Types;
import java.util.List;
import java.util.regex.Pattern;
+import org.apache.hop.core.DbCache;
+import org.apache.hop.core.DbCacheEntry;
import org.apache.hop.core.HopClientEnvironment;
import org.apache.hop.core.Result;
import org.apache.hop.core.exception.HopDatabaseBatchException;
@@ -787,6 +790,50 @@ class DatabaseTest {
return sb.toString();
}
+ @Test
+ void getQueryFieldsDropsARowQueriedAcrossAClear() throws Exception {
+ DbCache cache = DbCache.getInstance();
+ cache.clear(null);
+ when(meta.getName()).thenReturn("conn");
+ when(meta.supportsPreparedStatementMetadataRetrieval()).thenReturn(true);
+
+ Database db = spy(new Database(log, variables, meta));
+ db.setConnection(conn);
+ RowMeta fresh = new RowMeta();
+ fresh.addValueMeta(new ValueMetaString(columnName));
+ doAnswer(
+ invocation -> {
+ cache.clear(null);
+ return fresh;
+ })
+ .when(db)
+ .getQueryFieldsFromPreparedStatement(sql);
+
+ IRowMeta result = db.getQueryFields(sql, false);
+
+ assertEquals(columnName, result.getValueMeta(0).getName());
+ assertNull(cache.get(new DbCacheEntry("conn", sql)));
+ }
+
+ @Test
+ void getQueryFieldsStoresARowQueriedAfterAClear() throws Exception {
+ DbCache cache = DbCache.getInstance();
+ cache.clear(null);
+ when(meta.getName()).thenReturn("conn");
+ when(meta.supportsPreparedStatementMetadataRetrieval()).thenReturn(true);
+
+ Database db = spy(new Database(log, variables, meta));
+ db.setConnection(conn);
+ RowMeta fresh = new RowMeta();
+ fresh.addValueMeta(new ValueMetaString(columnName));
+ doReturn(fresh).when(db).getQueryFieldsFromPreparedStatement(sql);
+
+ IRowMeta result = db.getQueryFields(sql, false);
+
+ assertEquals(columnName, result.getValueMeta(0).getName());
+ assertEquals(columnName, cache.get(new DbCacheEntry("conn",
sql)).getValueMeta(0).getName());
+ }
+
@Test
void testGetQueryFieldsFromPreparedStatement() throws Exception {
when(rsMetaData.getColumnCount()).thenReturn(1);
diff --git
a/plugins/misc/lint/src/main/java/org/apache/hop/lint/BackgroundLintService.java
b/plugins/misc/lint/src/main/java/org/apache/hop/lint/BackgroundLintService.java
index e865f6838c..372afea143 100644
---
a/plugins/misc/lint/src/main/java/org/apache/hop/lint/BackgroundLintService.java
+++
b/plugins/misc/lint/src/main/java/org/apache/hop/lint/BackgroundLintService.java
@@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
import org.apache.hop.base.AbstractMeta;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.logging.ILogChannel;
@@ -73,6 +74,14 @@ public class BackgroundLintService {
*/
private final AtomicInteger generation = new AtomicInteger(0);
+ /**
+ * Bumped only by a Problems-tab force refresh. A pass remembers the value
it started with and
+ * does not publish once it has changed. Clearing the database cache from
anywhere else, the
+ * broom, the SQL editor, a table create or alter, or any DDL statement on
any connection, does
+ * not bump this, so a pass running at that moment still publishes.
+ */
+ private final AtomicInteger refreshGeneration = new AtomicInteger(0);
+
private String currentProjectPath;
private BackgroundLintService() {
@@ -178,6 +187,56 @@ public class BackgroundLintService {
}
}
+ /**
+ * Drop lint tracking for every open editor and re-lint all of them,
including the active one.
+ *
+ * <p>Called after the database cache has been cleared. Bumps this service's
own refresh
+ * generation, under the same lock a pass publishes with, so a pass that
started earlier cannot
+ * put the remarks from before the clear back. The active editor is
included: its Problems-tab
+ * check fills the tree, but it republishes lint markers only when lint is
part of pipeline
+ * verify, and a workflow check does not republish them at all.
+ *
+ * @param active the editor whose force-refresh button was pressed, or null
when every open editor
+ * is re-linted the same way
+ */
+ public void refreshOpenEditorsExcept(HopGuiAbstractGraph active) {
+ List<OpenEditor> open = new ArrayList<>();
+ for (HopGuiAbstractGraph graph : LintEditorGraphHelper.openGraphs()) {
+ open.add(new OpenEditor(graph,
LintEditorGraphHelper.getFilename(graph)));
+ }
+ refreshOpenEditorsExcept(
+ active, open, graph -> scheduleGraphLint((HopGuiAbstractGraph) graph,
true));
+ }
+
+ /**
+ * Bump the force-refresh generation, invalidate tracking for each open
editor, and schedule a
+ * lint of every one of them, including the active editor.
+ */
+ void refreshOpenEditorsExcept(Object active, List<OpenEditor> open,
Consumer<Object> schedule) {
+ synchronized (LintResultsManager.getInstance()) {
+ refreshGeneration.incrementAndGet();
+ }
+ log.logDetailed(
+ active == null
+ ? "Problems-tab force refresh re-lints every open editor"
+ : "Problems-tab force refresh re-lints the open editors, including
the active one");
+ if (open == null) {
+ return;
+ }
+ for (OpenEditor editor : open) {
+ if (editor == null || editor.graph() == null) {
+ continue;
+ }
+ tracker.invalidate(editor.filename());
+ if (schedule != null) {
+ schedule.accept(editor.graph());
+ }
+ }
+ }
+
+ /** An open editor, reduced to what a force refresh needs so a test does not
build a display. */
+ record OpenEditor(Object graph, String filename) {}
+
/**
* Called after the linter was switched off. Drops the findings and the
record of what was
* checked, and stops the passes still running from publishing or marking a
file as checked.
@@ -202,6 +261,11 @@ public class BackgroundLintService {
return generation.get();
}
+ /** The force-refresh generation a pass takes when it starts. */
+ int currentRefreshGeneration() {
+ return refreshGeneration.get();
+ }
+
/**
* Publish the findings of a pass, unless the project was switched or the
linter switched off
* since it started.
@@ -212,19 +276,35 @@ public class BackgroundLintService {
* @return false when the findings were dropped
*/
boolean publish(String filePath, List<LintResult> results, int
passGeneration) {
- return publish(filePath, results, passGeneration, false);
+ return publish(filePath, results, passGeneration, false,
currentRefreshGeneration());
+ }
+
+ /**
+ * {@link #publish}, and record the file as checked, unless a Problems-tab
force refresh started
+ * after {@code refreshGenerationAtStart} was read.
+ *
+ * @return false when the findings were dropped
+ */
+ boolean publishIfCurrent(
+ String filePath, List<LintResult> results, int passGeneration, int
refreshGenerationAtStart) {
+ return publish(filePath, results, passGeneration, true,
refreshGenerationAtStart);
}
/** {@link #publish}, and record the file as checked so an unchanged file is
not linted again. */
- private boolean publishChecked(String filePath, List<LintResult> results,
int passGeneration) {
- return publish(filePath, results, passGeneration, true);
+ private boolean publishChecked(
+ String filePath, List<LintResult> results, int passGeneration, int
refreshGenerationAtStart) {
+ return publish(filePath, results, passGeneration, true,
refreshGenerationAtStart);
}
private boolean publish(
- String filePath, List<LintResult> results, int passGeneration, boolean
markChecked) {
+ String filePath,
+ List<LintResult> results,
+ int passGeneration,
+ boolean markChecked,
+ int refreshGenerationAtStart) {
LintResultsManager manager = LintResultsManager.getInstance();
synchronized (manager) {
- if (!isCurrent(passGeneration)) {
+ if (!isCurrent(passGeneration) ||
!sameRefreshGeneration(refreshGenerationAtStart)) {
return false;
}
if (markChecked) {
@@ -251,6 +331,16 @@ public class BackgroundLintService {
return passGeneration == generation.get() && isEnabled();
}
+ /** False when a Problems-tab force refresh started during this pass. */
+ private boolean sameRefreshGeneration(int refreshGenerationAtStart) {
+ if (refreshGeneration.get() == refreshGenerationAtStart) {
+ return true;
+ }
+ log.logDetailed(
+ "Not publishing lint results because a Problems-tab force refresh
started during the check");
+ return false;
+ }
+
public void lintProjectAsync(
String projectPath, IHopMetadataProvider metadataProvider, IVariables
variables) {
if (!isEnabled() || Utils.isEmpty(projectPath)) {
@@ -259,6 +349,7 @@ public class BackgroundLintService {
int passGeneration = currentGeneration();
submit(
() -> {
+ int refreshGenerationAtStart = currentRefreshGeneration();
try {
HopLinter linter = new HopLinter();
List<String> allFiles =
@@ -278,9 +369,13 @@ public class BackgroundLintService {
return;
}
try {
+ // Taken before the check. A Problems-tab force refresh during
this pass bumps the
+ // refresh generation, and the remarks from before that
refresh must not be
+ // published.
+ int refreshGenerationAtCheck = currentRefreshGeneration();
List<LintResult> results =
lintFileForGui(filePath, linter, metadataProvider,
variables);
- if (publishChecked(filePath, results, passGeneration)) {
+ if (publishChecked(filePath, results, passGeneration,
refreshGenerationAtCheck)) {
accumulated.addAll(results);
}
} catch (Exception e) {
@@ -293,6 +388,11 @@ public class BackgroundLintService {
+ " issues in "
+ filesToCheck.size()
+ " files");
+ // A force refresh started while this pass ran and is already
rechecking the open
+ // editors. Repainting them from this pass would put the old
remarks back.
+ if (currentRefreshGeneration() != refreshGenerationAtStart) {
+ return;
+ }
LintProblemsBarManager.getInstance().refreshAllOpenEditors();
} catch (Exception e) {
log.logError("Error in background project linting", e);
@@ -332,6 +432,9 @@ public class BackgroundLintService {
submit(
() -> {
+ // Taken before the check. A Problems-tab force refresh during this
pass bumps the refresh
+ // generation, and the remarks from before that refresh must not be
published.
+ int refreshGenerationAtCheck = currentRefreshGeneration();
try {
// Switched off while this pass was waiting or running: do not
publish what it found.
if (!isEnabled()) {
@@ -345,7 +448,7 @@ public class BackgroundLintService {
List<LintResult> results =
lintSnapshot(snapshot, normalizedPath, metadataProvider,
variables);
- if (!publishChecked(filename, results, passGeneration)) {
+ if (!publishChecked(filename, results, passGeneration,
refreshGenerationAtCheck)) {
return;
}
Display display = graph.getDisplay();
@@ -462,13 +565,16 @@ public class BackgroundLintService {
* @return false when the findings were dropped
*/
boolean lintAndPublishFile(String filePath, int passGeneration) throws
HopException {
+ // Taken before the check. A Problems-tab force refresh during this pass
bumps the refresh
+ // generation, and the remarks from before that refresh must not be
published.
+ int refreshGenerationAtCheck = currentRefreshGeneration();
HopGui hopGui = HopGui.peekInstance();
IHopMetadataProvider metadataProvider = hopGui != null ?
hopGui.getMetadataProvider() : null;
IVariables variables = hopGui != null ? hopGui.getVariables() : null;
HopLinter linter = new HopLinter();
List<LintResult> results = lintFileForGui(filePath, linter,
metadataProvider, variables);
- if (!publishChecked(filePath, results, passGeneration)) {
+ if (!publishChecked(filePath, results, passGeneration,
refreshGenerationAtCheck)) {
return false;
}
log.logDetailed("Linted " + filePath + ": " + results.size() + " issues");
diff --git
a/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintEditorGraphHelper.java
b/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintEditorGraphHelper.java
index 4c172016ab..36edbb5af6 100644
---
a/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintEditorGraphHelper.java
+++
b/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintEditorGraphHelper.java
@@ -17,6 +17,8 @@
package org.apache.hop.lint;
import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.core.util.Utils;
import org.apache.hop.ui.hopgui.HopGui;
@@ -106,6 +108,29 @@ public final class LintEditorGraphHelper {
return null;
}
+ /** Every open pipeline or workflow editor. Empty when there is no GUI. */
+ public static List<HopGuiAbstractGraph> openGraphs() {
+ List<HopGuiAbstractGraph> graphs = new ArrayList<>();
+ // peekInstance: listing editors must not build a HopGui. On a headless
run that throws.
+ if (HopGui.peekInstance() == null) {
+ return graphs;
+ }
+ try {
+ ExplorerPerspective perspective = HopGui.getExplorerPerspective();
+ if (perspective == null) {
+ return graphs;
+ }
+ for (TabItemHandler item : perspective.getItems()) {
+ if (item.getTypeHandler() instanceof HopGuiAbstractGraph graph &&
!graph.isDisposed()) {
+ graphs.add(graph);
+ }
+ }
+ } catch (Exception e) {
+ LogChannel.GENERAL.logDetailed("Could not list open editors: " +
e.getMessage());
+ }
+ return graphs;
+ }
+
public static void scheduleAttachForFilename(String filename) {
if (!isLintableFilename(filename)) {
return;
diff --git
a/plugins/misc/lint/src/main/java/org/apache/hop/lint/ProblemsForceRefreshLintExtension.java
b/plugins/misc/lint/src/main/java/org/apache/hop/lint/ProblemsForceRefreshLintExtension.java
new file mode 100644
index 0000000000..eceae69b57
--- /dev/null
+++
b/plugins/misc/lint/src/main/java/org/apache/hop/lint/ProblemsForceRefreshLintExtension.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.lint;
+
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.extension.ExtensionPoint;
+import org.apache.hop.core.extension.IExtensionPoint;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.ui.hopgui.file.shared.HopGuiAbstractGraph;
+
+/**
+ * Re-lints every open editor after a Problems-tab force refresh.
+ *
+ * <p>The button clears the database cache and then checks the editor it
belongs to. That check
+ * fills the Problems tab, but it republishes lint markers only when lint is
included in pipeline
+ * verify, and a workflow check does not republish them. Linting every open
editor here, including
+ * the active one, replaces the markers from before the clear.
+ */
+@ExtensionPoint(
+ id = "ProblemsForceRefreshLintExtension",
+ extensionPointId = "ProblemsForceRefresh",
+ description = "Re-lints every open editor after a Problems-tab force
refresh")
+public class ProblemsForceRefreshLintExtension implements
IExtensionPoint<HopGuiAbstractGraph> {
+
+ @Override
+ public void callExtensionPoint(ILogChannel log, IVariables variables,
HopGuiAbstractGraph active)
+ throws HopException {
+ BackgroundLintService.getInstance().refreshOpenEditorsExcept(active);
+ }
+}
diff --git
a/plugins/misc/lint/src/test/java/org/apache/hop/lint/ProblemsForceRefreshLintTest.java
b/plugins/misc/lint/src/test/java/org/apache/hop/lint/ProblemsForceRefreshLintTest.java
new file mode 100644
index 0000000000..c04f88ad0a
--- /dev/null
+++
b/plugins/misc/lint/src/test/java/org/apache/hop/lint/ProblemsForceRefreshLintTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.lint;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.DbCache;
+import org.apache.hop.lint.BackgroundLintService.OpenEditor;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * A force refresh re-lints every open editor, including the active one, and a
pass that started
+ * before that refresh does not put the old remarks back. Clearing the
database cache on its own
+ * does not drop the pass.
+ */
+class ProblemsForceRefreshLintTest {
+
+ @AfterEach
+ void restore() {
+ LinterConfigPlugin config = LinterConfigPlugin.getInstance();
+ config.setLinterEnabled(true);
+ config.saveToHopConfig();
+ LintResultsManager.getInstance().clearResults();
+ DbCache.getInstance().clear(null);
+ }
+
+ @Test
+ void everyOpenEditorIsScheduledIncludingTheActiveOne() throws Exception {
+ BackgroundLintService service = BackgroundLintService.getInstance();
+ Object active = new Object();
+ Object other = new Object();
+ File activeFile = File.createTempFile("force-refresh-active", ".hpl");
+ File otherFile = File.createTempFile("force-refresh-other", ".hpl");
+ activeFile.deleteOnExit();
+ otherFile.deleteOnExit();
+ service.getTracker().markChecked(activeFile.getAbsolutePath());
+ service.getTracker().markChecked(otherFile.getAbsolutePath());
+ int generation = service.currentRefreshGeneration();
+
+ List<Object> scheduled = new ArrayList<>();
+ service.refreshOpenEditorsExcept(
+ active,
+ List.of(
+ new OpenEditor(active, activeFile.getAbsolutePath()),
+ new OpenEditor(other, otherFile.getAbsolutePath())),
+ scheduled::add);
+
+ assertEquals(generation + 1, service.currentRefreshGeneration());
+ assertEquals(List.of(active, other), scheduled);
+
assertTrue(service.getTracker().needsLinting(activeFile.getAbsolutePath()));
+ assertTrue(service.getTracker().needsLinting(otherFile.getAbsolutePath()));
+ }
+
+ @Test
+ void aPassStartedBeforeTheRefreshDoesNotPublish() {
+ enableLinter();
+ BackgroundLintService service = BackgroundLintService.getInstance();
+ LintResultsManager manager = LintResultsManager.getInstance();
+ manager.clearResults();
+ int generation = service.currentRefreshGeneration();
+ service.refreshOpenEditorsExcept(null, List.of(), null);
+
+ assertFalse(
+ service.publishIfCurrent(
+ "/tmp/force-refresh.hpl",
+ List.of(remark("too late")),
+ service.currentGeneration(),
+ generation));
+ assertTrue(manager.getResultsForFile("/tmp/force-refresh.hpl").isEmpty());
+ }
+
+ @Test
+ void aPassStartedAfterTheRefreshPublishes() {
+ enableLinter();
+ BackgroundLintService service = BackgroundLintService.getInstance();
+ LintResultsManager manager = LintResultsManager.getInstance();
+ manager.clearResults();
+ service.refreshOpenEditorsExcept(null, List.of(), null);
+ int generation = service.currentRefreshGeneration();
+
+ assertTrue(
+ service.publishIfCurrent(
+ "/tmp/force-refresh.hpl",
+ List.of(remark("current")),
+ service.currentGeneration(),
+ generation));
+ assertEquals(
+ "current",
manager.getResultsForFile("/tmp/force-refresh.hpl").get(0).getMessage());
+ }
+
+ @Test
+ void aCacheClearOnAnotherConnectionStillPublishes() {
+ enableLinter();
+ BackgroundLintService service = BackgroundLintService.getInstance();
+ LintResultsManager manager = LintResultsManager.getInstance();
+ manager.clearResults();
+ int generation = service.currentRefreshGeneration();
+ int cacheGeneration = DbCache.getInstance().getGeneration();
+ DbCache.getInstance().clear("some-other-connection");
+
+ assertNotEquals(cacheGeneration, DbCache.getInstance().getGeneration());
+ assertEquals(generation, service.currentRefreshGeneration());
+ assertTrue(
+ service.publishIfCurrent(
+ "/tmp/force-refresh.hpl",
+ List.of(remark("still current")),
+ service.currentGeneration(),
+ generation));
+ assertEquals(
+ "still current",
manager.getResultsForFile("/tmp/force-refresh.hpl").get(0).getMessage());
+ }
+
+ private static void enableLinter() {
+ LinterConfigPlugin config = LinterConfigPlugin.getInstance();
+ config.setLinterEnabled(true);
+ config.saveToHopConfig();
+ }
+
+ private static LintResult remark(String message) {
+ return new LintResult("HOP-CHECK", "Table output", "ERROR", message,
"/tmp/force-refresh.hpl");
+ }
+}
diff --git
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/delegates/HopGuiPipelineCheckDelegate.java
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/delegates/HopGuiPipelineCheckDelegate.java
index d9db46aab8..07a24c51bc 100644
---
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/delegates/HopGuiPipelineCheckDelegate.java
+++
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/delegates/HopGuiPipelineCheckDelegate.java
@@ -44,6 +44,7 @@ import org.apache.hop.ui.hopgui.HopGui;
import org.apache.hop.ui.hopgui.ToolbarFacade;
import org.apache.hop.ui.hopgui.file.IHopFileTypeHandler;
import org.apache.hop.ui.hopgui.file.pipeline.HopGuiPipelineGraph;
+import org.apache.hop.ui.hopgui.file.shared.ProblemsForceRefresh;
import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.CTabItem;
import org.eclipse.swt.graphics.Image;
@@ -61,6 +62,8 @@ public class HopGuiPipelineCheckDelegate {
private static final Class<?> PKG = HopGui.class;
public static final String GUI_PLUGIN_TOOLBAR_PARENT_ID =
"HopGuiPipelineCheckDelegate-ToolBar";
+ private static final String TOOLBAR_ITEM_FORCE_REFRESH =
+ "HopGuiPipelineCheckDelegate-Toolbar-10000-ForceRefresh";
private static final String TOOLBAR_ITEM_COLLAPSE_ALL =
"HopGuiPipelineCheckDelegate-Toolbar-10010-CollapseAll";
private static final String TOOLBAR_ITEM_EXPAND_ALL =
@@ -164,6 +167,18 @@ public class HopGuiPipelineCheckDelegate {
}
}
+ @GuiToolbarElement(
+ root = GUI_PLUGIN_TOOLBAR_PARENT_ID,
+ id = TOOLBAR_ITEM_FORCE_REFRESH,
+ toolTip =
"i18n:org.apache.hop.ui.hopgui:PipelineGraph.Check.ForceRefresh.Tooltip",
+ image = "ui/images/force-refresh.svg")
+ public void forceRefresh() {
+ // Same shape as the execution perspective: drop the caches, then load
again. The check of
+ // this editor fills the Problems tab. Every open editor, including this
one, is re-linted in
+ // the background so its markers are replaced.
+ ProblemsForceRefresh.refresh(pipelineGraph, this::checkPipeline);
+ }
+
@GuiToolbarElement(
root = GUI_PLUGIN_TOOLBAR_PARENT_ID,
id = TOOLBAR_ITEM_COLLAPSE_ALL,
diff --git
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/shared/ProblemsForceRefresh.java
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/shared/ProblemsForceRefresh.java
new file mode 100644
index 0000000000..61382f2b78
--- /dev/null
+++
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/shared/ProblemsForceRefresh.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.ui.hopgui.file.shared;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.DbCache;
+import org.apache.hop.core.extension.ExtensionPointHandler;
+import org.apache.hop.core.extension.HopExtensionPoint;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.ui.hopgui.HopGui;
+import org.apache.hop.ui.hopgui.file.pipeline.HopGuiPipelineGraph;
+import org.apache.hop.ui.hopgui.perspective.TabItemHandler;
+import org.apache.hop.ui.hopgui.perspective.explorer.ExplorerPerspective;
+
+/**
+ * Clears cached database metadata and asks for a fresh check of the open
editors.
+ *
+ * <p>The Problems tab of a pipeline and of a workflow both call this. The
database cache is
+ * process-wide, so the click also drops the field lists of every open
pipeline. The editor whose
+ * button was pressed is re-checked by the caller. Every open editor,
including that one, is handed
+ * to {@link HopExtensionPoint#ProblemsForceRefresh} so the linter replaces
markers the check itself
+ * does not republish.
+ */
+public final class ProblemsForceRefresh {
+
+ private ProblemsForceRefresh() {}
+
+ /**
+ * Clear the database cache and the open pipelines' field caches, then
re-check.
+ *
+ * @param active the editor whose Problems tab was clicked
+ * @param recheckActive runs the existing check of that editor, after the
caches are empty
+ */
+ public static void refresh(HopGuiAbstractGraph active, Runnable
recheckActive) {
+ refresh(openPipelineMetas(), () -> notifyOthers(active), recheckActive);
+ }
+
+ /**
+ * The cache half of {@link #refresh(HopGuiAbstractGraph, Runnable)}, with
the open pipelines and
+ * the "other editors" step supplied by the caller. Tests use this so they
do not need a display.
+ */
+ static void refresh(
+ List<PipelineMeta> openPipelines, Runnable refreshOthers, Runnable
recheckActive) {
+ DbCache.getInstance().clear(null);
+ if (openPipelines != null) {
+ for (PipelineMeta pipelineMeta : openPipelines) {
+ if (pipelineMeta != null) {
+ pipelineMeta.clearCaches();
+ }
+ }
+ }
+ if (refreshOthers != null) {
+ refreshOthers.run();
+ }
+ if (recheckActive != null) {
+ recheckActive.run();
+ }
+ }
+
+ private static void notifyOthers(HopGuiAbstractGraph active) {
+ try {
+ ExtensionPointHandler.callExtensionPoint(
+ LogChannel.GENERAL,
+ Variables.getADefaultVariableSpace(),
+ HopExtensionPoint.ProblemsForceRefresh.id,
+ active);
+ } catch (Exception e) {
+ LogChannel.GENERAL.logError(
+ "Could not refresh the open editors after clearing the database
cache", e);
+ }
+ }
+
+ private static List<PipelineMeta> openPipelineMetas() {
+ List<PipelineMeta> pipelines = new ArrayList<>();
+ // peekInstance: asking for the perspective must not build a HopGui. On a
headless run that
+ // throws instead of returning null.
+ if (HopGui.peekInstance() == null) {
+ return pipelines;
+ }
+ try {
+ ExplorerPerspective perspective = HopGui.getExplorerPerspective();
+ if (perspective == null) {
+ return pipelines;
+ }
+ for (TabItemHandler item : perspective.getItems()) {
+ if (item.getTypeHandler() instanceof HopGuiPipelineGraph graph
+ && !graph.isDisposed()
+ && graph.getPipelineMeta() != null) {
+ pipelines.add(graph.getPipelineMeta());
+ }
+ }
+ } catch (Exception e) {
+ LogChannel.GENERAL.logDetailed("Could not list open pipelines: " +
e.getMessage());
+ }
+ return pipelines;
+ }
+}
diff --git
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/workflow/delegates/HopGuiWorkflowCheckDelegate.java
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/workflow/delegates/HopGuiWorkflowCheckDelegate.java
index cc8fc28df1..4ccd73cece 100644
---
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/workflow/delegates/HopGuiWorkflowCheckDelegate.java
+++
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/workflow/delegates/HopGuiWorkflowCheckDelegate.java
@@ -40,6 +40,7 @@ import org.apache.hop.ui.core.gui.IToolbarContainer;
import org.apache.hop.ui.hopgui.HopGui;
import org.apache.hop.ui.hopgui.ToolbarFacade;
import org.apache.hop.ui.hopgui.file.IHopFileTypeHandler;
+import org.apache.hop.ui.hopgui.file.shared.ProblemsForceRefresh;
import org.apache.hop.ui.hopgui.file.workflow.HopGuiWorkflowGraph;
import org.apache.hop.workflow.WorkflowMeta;
import org.apache.hop.workflow.action.ActionMeta;
@@ -61,6 +62,8 @@ public class HopGuiWorkflowCheckDelegate {
private static final Class<?> PKG = HopGui.class;
public static final String GUI_PLUGIN_TOOLBAR_PARENT_ID =
"HopGuiWorkflowCheckDelegate-ToolBar";
+ private static final String TOOLBAR_ITEM_FORCE_REFRESH =
+ "HopGuiWorkflowCheckDelegate-Toolbar-10000-ForceRefresh";
private static final String TOOLBAR_ITEM_COLLAPSE_ALL =
"HopGuiWorkflowCheckDelegate-Toolbar-10010-CollapseAll";
private static final String TOOLBAR_ITEM_EXPAND_ALL =
@@ -165,6 +168,18 @@ public class HopGuiWorkflowCheckDelegate {
}
}
+ @GuiToolbarElement(
+ root = GUI_PLUGIN_TOOLBAR_PARENT_ID,
+ id = TOOLBAR_ITEM_FORCE_REFRESH,
+ toolTip =
"i18n:org.apache.hop.ui.hopgui:PipelineGraph.Check.ForceRefresh.Tooltip",
+ image = "ui/images/force-refresh.svg")
+ public void forceRefresh() {
+ // Same shape as the execution perspective: drop the caches, then load
again. The check of
+ // this editor fills the Problems tab. Every open editor, including this
one, is re-linted in
+ // the background so its markers are replaced.
+ ProblemsForceRefresh.refresh(workflowGraph, this::checkWorkflow);
+ }
+
@GuiToolbarElement(
root = GUI_PLUGIN_TOOLBAR_PARENT_ID,
id = TOOLBAR_ITEM_COLLAPSE_ALL,
diff --git
a/ui/src/main/resources/org/apache/hop/ui/hopgui/messages/messages_en_US.properties
b/ui/src/main/resources/org/apache/hop/ui/hopgui/messages/messages_en_US.properties
index 37e33ec483..a3a4317f44 100644
---
a/ui/src/main/resources/org/apache/hop/ui/hopgui/messages/messages_en_US.properties
+++
b/ui/src/main/resources/org/apache/hop/ui/hopgui/messages/messages_en_US.properties
@@ -195,6 +195,7 @@ LogBrowser.Filter.CaseSensitive.Tooltip=Match the filter
text with case sensitiv
LogBrowser.Filter.Exclude.Label=Exclude
LogBrowser.Filter.Exclude.Tooltip=Hide log lines that match the filter text
(exclude from the log view)
PipelineGraph.Check.Tab.Name=Problems
+PipelineGraph.Check.ForceRefresh.Tooltip=Force refresh (clear database cache
and check again)
PipelineGraph.Check.PipelineRemarks=Pipeline
PipelineGraph.Check.ErrorCheckingPipeline.Exception=Error checking pipeline:
{0}
PipelineGraph.Check.ErrorCheckingPipeline.Message=Error checking pipeline
diff --git
a/ui/src/test/java/org/apache/hop/ui/hopgui/file/shared/ProblemsForceRefreshTest.java
b/ui/src/test/java/org/apache/hop/ui/hopgui/file/shared/ProblemsForceRefreshTest.java
new file mode 100644
index 0000000000..1e4737e2fe
--- /dev/null
+++
b/ui/src/test/java/org/apache/hop/ui/hopgui/file/shared/ProblemsForceRefreshTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.ui.hopgui.file.shared;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.spy;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.DbCache;
+import org.apache.hop.core.DbCacheEntry;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.junit.jupiter.api.Test;
+
+/** The Problems tab force refresh drops cached database metadata before it
checks again. */
+class ProblemsForceRefreshTest {
+
+ @Test
+ void oneClickClearsTheCachesThenRefreshesTheOtherEditorsThenRechecks() {
+ DbCache cache = DbCache.getInstance();
+ cache.clear(null);
+ int generation = cache.getGeneration();
+ cache.put(new DbCacheEntry("conn", "select * from t"), row("old"));
+
+ PipelineMeta openPipeline = spy(new PipelineMeta());
+ List<String> order = new ArrayList<>();
+ doAnswer(
+ invocation -> {
+ order.add("caches");
+ return null;
+ })
+ .when(openPipeline)
+ .clearCaches();
+
+ ProblemsForceRefresh.refresh(
+ List.of(openPipeline),
+ () -> {
+ order.add("others");
+ assertNull(cache.get(new DbCacheEntry("conn", "select * from t")));
+ },
+ () -> order.add("recheck"));
+
+ assertNotEquals(generation, cache.getGeneration());
+ assertEquals(List.of("caches", "others", "recheck"), order);
+ }
+
+ private static RowMeta row(String name) {
+ RowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString(name));
+ return rowMeta;
+ }
+}
