hansva commented on code in PR #9545: URL: https://github.com/apache/hop/pull/9545#discussion_r4614486331
########## plugins/tech/datahub/src/main/java/org/apache/hop/lineage/datahub/DataHubPublisher.java: ########## @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.lineage.datahub; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import lombok.Setter; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSelectInfo; +import org.apache.commons.vfs2.FileSelector; +import org.apache.commons.vfs2.FileType; +import org.apache.hop.base.AbstractMeta; +import org.apache.hop.core.Const; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.logging.ILogChannel; +import org.apache.hop.core.parameters.UnknownParamException; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.lineage.LineageDefinitions; +import org.apache.hop.lineage.hub.LineageSinkTarget; +import org.apache.hop.lineage.metadata.LineageSink; +import org.apache.hop.lineage.model.LineageEvent; +import org.apache.hop.lineage.spi.LineageSinkHealth; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.workflow.WorkflowMeta; + +/** + * Publishes the pipelines and workflows of a folder to DataHub without running them: each as a flow + * with its transforms or actions as tasks, the tables and files they declare as their inputs and + * outputs, and the layouts of those files as dataset schemas. It writes through a DataHub {@link + * LineageSink}, so the entities are the ones the runs report to. + * + * <p>With {@code includingDocumentation} set it also publishes what {@code hop doc} writes for each + * pipeline and workflow as a DataHub document (DataHub 1.7 and later), related to its flow and + * tasks, under a document of the project. + */ +public final class DataHubPublisher { + + /** How long to wait for DataHub to accept what was published. */ + static final long DELIVERY_TIMEOUT_MS = 120_000L; + + /** How long to wait for DataHub to take the first document, before the others are sent. */ + static final long FIRST_DOCUMENT_TIMEOUT_MS = 30_000L; + + /** The name of the project, set when a project is used. */ + static final String VAR_PROJECT_NAME = "HOP_PROJECT_NAME"; + + /** + * What a publication did. + * + * @param pipelines the pipelines published + * @param workflows the workflows published + * @param unreadable the files that could not be read, and were skipped + * @param documentationNotPublished why the documentation was not published, or null + * @param health how the delivery to DataHub went + */ + public record Result( + int pipelines, + int workflows, + List<String> unreadable, + String documentationNotPublished, + LineageSinkHealth health) { + + /** Whether everything was read and DataHub accepted all of it. */ + public boolean isSuccess() { + return unreadable.isEmpty() + && documentationNotPublished == null + && health != null + && health.waiting() == 0 + && health.rejected() == 0 + && health.dropped() == 0; + } + } + + private final IVariables variables; + private final IHopMetadataProvider metadataProvider; + private final ILogChannel log; + + /** + * Whether to publish what {@code hop doc} writes for each pipeline and workflow as well, as + * DataHub documents: DataHub 1.7 or later, an earlier version refuses them. + */ + @Setter private boolean includingDocumentation; + + /** Whether the documentation lists the parameters, as {@code hop doc -ip} does. */ + @Setter private boolean includingParameters; + + /** Whether the documentation has the text of the notes, as {@code hop doc -in} does. */ + @Setter private boolean includingNotes; + + public DataHubPublisher( + IVariables variables, IHopMetadataProvider metadataProvider, ILogChannel log) { + this.variables = variables; + this.metadataProvider = metadataProvider; + this.log = log; + } + + /** + * Publishes every pipeline and workflow in {@code folder} and its subfolders. + * + * @param sinkName the name of a lineage sink of type DataHub + * @param folder the folder, variables allowed + * @throws HopException when the sink can't be used, or the folder doesn't exist + */ + public Result publish(String sinkName, String folder) throws HopException { + DataHubLineageSink sink = startSink(sinkName); + int pipelines = 0; + int workflows = 0; + List<String> unreadable = new ArrayList<>(); + List<LineageEvent> documents = new ArrayList<>(); + String documentationNotPublished = null; + LineageSinkHealth health; + try { + FileObject root = folder(folder); + DocumentationWriter documentation = includingDocumentation ? documentationWriter() : null; + String project = variables.getVariable(VAR_PROJECT_NAME); + for (FileObject file : hopFiles(root)) { + String filename = HopVfs.getFilename(file); + try { + String relativeFile = root.getName().getRelativeName(file.getName()); + if (filename.endsWith(".hpl")) { + PipelineMeta meta = new PipelineMeta(filename, metadataProvider, variables); + List<LineageEvent> events = + LineageDefinitions.ofPipeline(meta, variablesOf(meta, variables), log); + if (documentation != null) { + documents.add(documentation.pipeline(events.get(0), meta, relativeFile, project)); Review Comment: **A documentation failure drops the pipeline's lineage.** The Markdown documentation is built before `sink.accept(events)`. If `documentation.pipeline(...)` throws for one file (say a bad parameter definition), the catch skips `sink.accept`, so that pipeline's flow, tasks and datasets never reach DataHub, and the file is reported as unreadable. Without `-d` the same file publishes fine. Suggest calling `sink.accept(events)` first, or putting the documentation step in its own try/catch that only logs a warning. The workflow branch looks like it has the same pattern. ########## engine/src/main/java/org/apache/hop/lineage/hub/LineageHub.java: ########## @@ -124,12 +384,18 @@ public void environmentReady() { sinksInitialized.set(false); sinkInitFailed.set(false); sinks = List.of(); + sinkIds.clear(); + sinkInitError = null; + routedSinks.clear(); Review Comment: **Routed sinks are dropped without being shut down.** `routedSinks.clear()` forgets the sinks that runs started from their run configuration, but leaves their dispatcher threads, HTTP clients and spools running. After a second `HopEnvironment.init` (GUI/server re-init, tests), the next routed event starts a second sink for the same key, and both may replay the same spool directory. Please shut down each routed sink before clearing the map. ########## plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/textfileoutput/TextFileOutputMeta.java: ########## @@ -683,6 +689,86 @@ String[] getFiles( } } + /** + * The layout of the file this transform writes: the output fields when some are listed, the whole + * row otherwise. + * + * @param outputRowMeta the row the transform writes; null or empty when it is not known, then + * only listed fields give a layout, with the types they are listed with + */ + public FileIoContentSchema writtenContentSchema(IRowMeta outputRowMeta) { + List<FileIoTabularColumn> cols = new ArrayList<>(); + if (outputFields != null && !outputFields.isEmpty()) { + for (TextFileField f : outputFields) { + String typeDesc = f.getTypeDesc(); + if (outputRowMeta != null) { + IValueMeta v = outputRowMeta.searchValueMeta(f.getName()); + if (v != null) { + typeDesc = v.getTypeDesc(); + } + } + cols.add( + new FileIoTabularColumn( + f.getName(), + typeDesc, + f.getLength(), + f.getPrecision(), + null, + FileIoPathSyntax.NONE, + false)); + } + } else if (outputRowMeta != null) { + for (int i = 0; i < outputRowMeta.size(); i++) { + IValueMeta v = outputRowMeta.getValueMeta(i); + cols.add( + new FileIoTabularColumn( + v.getName(), + v.getTypeDesc(), + v.getLength(), + v.getPrecision(), + null, + FileIoPathSyntax.DELIMITED, + false)); + } + } + if (cols.isEmpty()) { + return null; + } + return new FileIoContentSchema("delimited", cols, List.of()); + } + + /** + * The file a run writes first, with its layout; none when the name comes from a field, or holds + * the date or time of the run, so no publication names a file no run writes. + */ + @Override + public List<IDeclaresLineage.Declaration> declaredLineage( + IVariables variables, IRowMeta inputRowMeta) { + boolean dated = + (fileSettings.isSpecifyingFormat() && !Utils.isEmpty(fileSettings.getDateTimeFormat())) + || (!fileSettings.isSpecifyingFormat() + && (fileSettings.isDateInFilename() || fileSettings.isTimeInFilename())); + if (fileNameInField || Utils.isEmpty(fileSettings.fileName) || dated) { + return List.of(); + } + String filename = + buildFilename( Review Comment: **Phantom `_null` file.** `partNr` is passed as `null`, and the filename builder does `retval += "_" + partNr` when *Include partition nr in filename* is on. So `/out/orders` + `csv` is declared as `/out/orders_null.csv`, a file no run writes. Date or time in the filename have a similar problem: today's date or time is put into the declared name. For declared (design-time) lineage I'd declare the base name, or a pattern, and leave out the per-run suffixes. ########## plugins/tech/datahub/src/main/java/org/apache/hop/lineage/datahub/DataHubPublisher.java: ########## @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.lineage.datahub; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import lombok.Setter; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSelectInfo; +import org.apache.commons.vfs2.FileSelector; +import org.apache.commons.vfs2.FileType; +import org.apache.hop.base.AbstractMeta; +import org.apache.hop.core.Const; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.logging.ILogChannel; +import org.apache.hop.core.parameters.UnknownParamException; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.lineage.LineageDefinitions; +import org.apache.hop.lineage.hub.LineageSinkTarget; +import org.apache.hop.lineage.metadata.LineageSink; +import org.apache.hop.lineage.model.LineageEvent; +import org.apache.hop.lineage.spi.LineageSinkHealth; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.workflow.WorkflowMeta; + +/** + * Publishes the pipelines and workflows of a folder to DataHub without running them: each as a flow + * with its transforms or actions as tasks, the tables and files they declare as their inputs and + * outputs, and the layouts of those files as dataset schemas. It writes through a DataHub {@link + * LineageSink}, so the entities are the ones the runs report to. + * + * <p>With {@code includingDocumentation} set it also publishes what {@code hop doc} writes for each + * pipeline and workflow as a DataHub document (DataHub 1.7 and later), related to its flow and + * tasks, under a document of the project. + */ +public final class DataHubPublisher { + + /** How long to wait for DataHub to accept what was published. */ + static final long DELIVERY_TIMEOUT_MS = 120_000L; + + /** How long to wait for DataHub to take the first document, before the others are sent. */ + static final long FIRST_DOCUMENT_TIMEOUT_MS = 30_000L; + + /** The name of the project, set when a project is used. */ + static final String VAR_PROJECT_NAME = "HOP_PROJECT_NAME"; + + /** + * What a publication did. + * + * @param pipelines the pipelines published + * @param workflows the workflows published + * @param unreadable the files that could not be read, and were skipped + * @param documentationNotPublished why the documentation was not published, or null + * @param health how the delivery to DataHub went + */ + public record Result( + int pipelines, + int workflows, + List<String> unreadable, + String documentationNotPublished, + LineageSinkHealth health) { + + /** Whether everything was read and DataHub accepted all of it. */ + public boolean isSuccess() { + return unreadable.isEmpty() + && documentationNotPublished == null + && health != null + && health.waiting() == 0 + && health.rejected() == 0 + && health.dropped() == 0; + } + } + + private final IVariables variables; + private final IHopMetadataProvider metadataProvider; + private final ILogChannel log; + + /** + * Whether to publish what {@code hop doc} writes for each pipeline and workflow as well, as + * DataHub documents: DataHub 1.7 or later, an earlier version refuses them. + */ + @Setter private boolean includingDocumentation; + + /** Whether the documentation lists the parameters, as {@code hop doc -ip} does. */ + @Setter private boolean includingParameters; + + /** Whether the documentation has the text of the notes, as {@code hop doc -in} does. */ + @Setter private boolean includingNotes; + + public DataHubPublisher( + IVariables variables, IHopMetadataProvider metadataProvider, ILogChannel log) { + this.variables = variables; + this.metadataProvider = metadataProvider; + this.log = log; + } + + /** + * Publishes every pipeline and workflow in {@code folder} and its subfolders. + * + * @param sinkName the name of a lineage sink of type DataHub + * @param folder the folder, variables allowed + * @throws HopException when the sink can't be used, or the folder doesn't exist + */ + public Result publish(String sinkName, String folder) throws HopException { + DataHubLineageSink sink = startSink(sinkName); + int pipelines = 0; + int workflows = 0; + List<String> unreadable = new ArrayList<>(); + List<LineageEvent> documents = new ArrayList<>(); + String documentationNotPublished = null; + LineageSinkHealth health; + try { + FileObject root = folder(folder); + DocumentationWriter documentation = includingDocumentation ? documentationWriter() : null; + String project = variables.getVariable(VAR_PROJECT_NAME); + for (FileObject file : hopFiles(root)) { + String filename = HopVfs.getFilename(file); + try { + String relativeFile = root.getName().getRelativeName(file.getName()); + if (filename.endsWith(".hpl")) { Review Comment: **`.HPL` files are loaded as workflows.** The file selector lowercases the extension, but this check is case-sensitive, so `LOAD.HPL` goes to the `WorkflowMeta` branch, fails, and is reported as unreadable. Use `filename.toLowerCase(Locale.ROOT).endsWith(".hpl")`, or reuse the extension the selector already computed. ########## plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileInputMeta.java: ########## @@ -1100,4 +1104,38 @@ public void convertLegacyXml(Node node) { } } } + + /** + * The files this transform reads, as they are now, with the layout of its fields; none when it + * takes the file names from another transform. + */ + @Override + public List<IDeclaresLineage.Declaration> declaredLineage( + IVariables variables, IRowMeta inputRowMeta) { + if (getFileInput().isAcceptingFilenames()) { + return List.of(); + } + IRowMeta layout = new RowMeta(); + for (TextFileInputField field : inputFields) { + try { + IValueMeta value = ValueMetaFactory.createValueMeta(field.getName(), field.getType()); + value.setLength(field.getLength()); + value.setPrecision(field.getPrecision()); + layout.addValueMeta(value); + } catch (HopPluginException e) { + // A type that is not installed: the layout goes without it + } + } + List<IDeclaresLineage.Declaration> declarations = new ArrayList<>(); + for (String path : getFilePaths(variables)) { + String extension = path.contains(".") ? path.substring(path.lastIndexOf('.') + 1) : null; Review Comment: **The file extension is taken from the full path.** `/data/v1.2/customers` gives `2/customers` as the file format. Take it from the base name instead (e.g. `FilenameUtils.getExtension`, or the VFS `FileName.getExtension()`). ########## engine/src/main/java/org/apache/hop/lineage/LineageSinkRoutes.java: ########## @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.lineage; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; +import org.apache.hop.core.logging.ILogChannel; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.lineage.hub.LineageHub; +import org.apache.hop.lineage.metadata.LineageSink; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.config.PipelineRunConfiguration; +import org.apache.hop.pipeline.engine.IPipelineEngine; +import org.apache.hop.pipeline.transform.ITransform; +import org.apache.hop.workflow.WorkflowMeta; +import org.apache.hop.workflow.action.IAction; +import org.apache.hop.workflow.config.WorkflowRunConfiguration; +import org.apache.hop.workflow.engine.IWorkflowEngine; + +/** + * The lineage sinks a run reports to, besides the ones configured with variables: the sink its run + * configuration selects, and the sinks of the run that started it. Worked out once per run, when it + * is first asked, and kept with the run. The run holds those sinks in the {@link LineageHub} until + * it ends and calls {@link #release(IPipelineEngine)}. + */ +public final class LineageSinkRoutes { + + /** Where a run keeps its sinks, in its extension data. */ + static final String EXTENSION_DATA_KEY = "lineage.sinks"; + + /** Set in a run's extension data once it let go of its sinks. */ + static final String RELEASED_KEY = "lineage.sinks.released"; + + private LineageSinkRoutes() {} + + /** + * Set to {@code Y} on a pipeline run that runs a unit test, and on the runs it starts (the + * testing plugin's {@code __UnitTest_Run__}). + */ + public static final String VAR_RUN_UNIT_TEST = "__UnitTest_Run__"; + + /** + * The keys of the sinks a pipeline run reports to; empty when there are none, or when the run + * runs a unit test. Such a run runs a copy of the pipeline, with data sets in place of transforms + * and no file name: its lineage is not the pipeline's. Its sinks get only the test's results, see + * {@link #ofUnitTest}. + */ + public static Set<String> of(IPipelineEngine<PipelineMeta> pipeline) { + return ofRun(pipeline, LineageHub.getInstance()); + } + + static Set<String> ofRun(IPipelineEngine<PipelineMeta> pipeline, LineageHub hub) { + // Held now, so they are released when the run ends, like the sinks of any run + Set<String> sinks = of(pipeline, hub); + return isUnitTestRun(pipeline) ? Set.of() : sinks; + } + + /** The keys of the sinks the results of the unit test a pipeline run runs report to. */ + public static Set<String> ofUnitTest(IPipelineEngine<PipelineMeta> pipeline) { + return of(pipeline, LineageHub.getInstance()); + } + + static boolean isUnitTestRun(IPipelineEngine<PipelineMeta> pipeline) { + return pipeline != null && "Y".equalsIgnoreCase(pipeline.getVariable(VAR_RUN_UNIT_TEST)); + } + + /** The keys of the sinks a workflow run reports to; empty when there are none. */ + public static Set<String> of(IWorkflowEngine<WorkflowMeta> workflow) { + return of(workflow, LineageHub.getInstance()); + } + + /** + * Whether lineage is observed for a pipeline run: lineage is enabled with variables, or the run + * reports to a sink of its own. Callers that do extra work only to fill in an event check this. + */ + public static boolean isEnabled(IPipelineEngine<PipelineMeta> pipeline) { + return LineageHub.getInstance().isEnabled() || !of(pipeline).isEmpty(); + } + + /** Whether lineage is observed for a workflow run; see {@link #isEnabled(IPipelineEngine)}. */ + public static boolean isEnabled(IWorkflowEngine<WorkflowMeta> workflow) { + return LineageHub.getInstance().isEnabled() || !of(workflow).isEmpty(); + } + + /** + * Whether lineage is observed for the run of a transform; see {@link + * #isEnabled(IPipelineEngine)}. + */ + public static boolean isEnabled(ITransform transform) { + return isEnabled(transform == null ? null : transform.getPipeline()); + } + + static Set<String> of(IPipelineEngine<PipelineMeta> pipeline, LineageHub hub) { + if (pipeline == null) { + return Set.of(); + } + return cached( + pipeline.getExtensionDataMap(), + hub, + () -> { + Set<String> parents = parentSinks(pipeline, hub); + PipelineRunConfiguration runConfiguration = pipeline.getPipelineRunConfiguration(); + String own = + useSink( + runConfiguration == null ? null : runConfiguration.getLineageSinkName(), + pipeline, + pipeline.getMetadataProvider(), + pipeline.getLogChannel(), + hub); + return withParents(parents, own, hub); + }); + } + + static Set<String> of(IWorkflowEngine<WorkflowMeta> workflow, LineageHub hub) { + if (workflow == null) { + return Set.of(); + } + return cached( + workflow.getExtensionDataMap(), + hub, + () -> { + Set<String> parents = parentSinks(workflow, hub); + WorkflowRunConfiguration runConfiguration = workflow.getWorkflowRunConfiguration(); + String own = + useSink( + runConfiguration == null ? null : runConfiguration.getLineageSinkName(), + workflow, + workflow.getMetadataProvider(), + workflow.getLogChannel(), + hub); + return withParents(parents, own, hub); + }); + } + + /** + * Lets go of the sinks a pipeline run held, once it ended and emitted its last event: a sink that + * was replaced by a newer version of itself can then be shut down. + */ + public static void release(IPipelineEngine<PipelineMeta> pipeline) { + release(pipeline == null ? null : pipeline.getExtensionDataMap(), LineageHub.getInstance()); + } + + /** Lets go of the sinks a workflow run held; see {@link #release(IPipelineEngine)}. */ + public static void release(IWorkflowEngine<WorkflowMeta> workflow) { + release(workflow == null ? null : workflow.getExtensionDataMap(), LineageHub.getInstance()); + } + + static void release(Map<String, Object> extensionData, LineageHub hub) { + if (extensionData == null) { + return; + } + Set<String> sinks; + synchronized (extensionData) { + if (!(extensionData.get(EXTENSION_DATA_KEY) instanceof Set<?> known) + || extensionData.putIfAbsent(RELEASED_KEY, Boolean.TRUE) != null) { + return; + } + @SuppressWarnings("unchecked") + Set<String> held = (Set<String>) known; + sinks = held; + } + hub.release(sinks); + } + + /** + * The sinks of the run that started this one and the run's own, each held once for this run, as + * {@link #release} lets go of each once. The parent's are safe to hold now: the parent holds them + * until it ends, after this run. + * + * @param own the run's own sink, which {@link LineageHub#useSink} already held, or null + */ + private static Set<String> withParents(Set<String> parentSinks, String own, LineageHub hub) { + Set<String> inherited = new LinkedHashSet<>(parentSinks); + if (own != null) { + inherited.remove(own); + } + hub.hold(inherited); + Set<String> sinks = new LinkedHashSet<>(parentSinks); + if (own != null) { + sinks.add(own); + } + return sinks; + } + + private static Set<String> parentSinks(IPipelineEngine<PipelineMeta> pipeline, LineageHub hub) { + if (pipeline.getParentPipeline() != null) { + return of(pipeline.getParentPipeline(), hub); + } + if (pipeline.getParentWorkflow() != null) { + return of(pipeline.getParentWorkflow(), hub); + } + return of(actionWorkflow(pipeline.getParent()), hub); + } + + private static Set<String> parentSinks(IWorkflowEngine<WorkflowMeta> workflow, LineageHub hub) { + if (workflow.getParentPipeline() != null) { + return of(workflow.getParentPipeline(), hub); + } + if (workflow.getParentWorkflow() != null) { + return of(workflow.getParentWorkflow(), hub); + } + return of(actionWorkflow(workflow.getParent()), hub); + } + + /** + * The workflow of an action that started a run without naming its workflow as the parent, such as + * the pipelines Run pipeline unit tests starts. + */ + private static IWorkflowEngine<WorkflowMeta> actionWorkflow(Object parent) { + return parent instanceof IAction action ? action.getParentWorkflow() : null; + } + + /** + * Uses and holds the sink of a run configuration. A sink that can't be loaded or resolved is + * logged on the run and left out: lineage must not stop a run. + * + * @return the key of the sink, or null when there is none + */ + private static String useSink( + String sinkName, + IVariables variables, + IHopMetadataProvider metadataProvider, + ILogChannel log, + LineageHub hub) { + String name = variables.resolve(sinkName); + if (Utils.isEmpty(name) || metadataProvider == null) { + return null; + } + try { + LineageSink sink = metadataProvider.getSerializer(LineageSink.class).load(name); + if (sink == null) { + throw new IllegalArgumentException("lineage sink '" + name + "' does not exist"); + } + String key = hub.useSink(sink.toTarget(variables, metadataProvider)); + String failure = hub.failure(key); + if (failure != null && log != null) { + log.logError( + "Lineage sink '" + + name + + "' failed to start (" + + failure + + "). It is tried again a minute after it failed; until then this run's lineage" + + " doesn't reach it."); + } + return key; + } catch (Exception e) { + if (log != null) { + log.logError("Unable to use lineage sink '" + name + "', this run doesn't report to it", e); + } + return null; + } + } + + private static Set<String> cached( + Map<String, Object> extensionData, LineageHub hub, Supplier<Set<String>> resolver) { + if (extensionData == null) { + return Collections.unmodifiableSet(resolver.get()); Review Comment: **Holds that are never released when there is no extension data map.** With `extensionData == null`, `resolver.get()` runs on every call and takes a new hub hold each time, but nothing records the set, so `release()` never gives it back. With an engine or mock whose `getExtensionDataMap()` returns null, hold counts only grow, and an edited sink's old version is never retired. Either don't take holds on this path, or fall back to a per-run map. ########## plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/ValidatePipelineUnitTestExtensionPoint.java: ########## @@ -73,17 +81,17 @@ public void callExtensionPoint( // If the pipeline has a variable set with the unit test in it, we're dealing with a unit test // situation. // - PipelineUnitTest unitTest = - metadataProvider.getSerializer(PipelineUnitTest.class).load(unitTestName); + unitTest = metadataProvider.getSerializer(PipelineUnitTest.class).load(unitTestName); - final List<UnitTestResult> results = new ArrayList<>(); pipeline.getExtensionDataMap().put(DataSetConst.UNIT_TEST_RESULTS, results); // Validate execution results with what's in the data sets... // int errors = DataSetConst.validateTransformResultAgainstUnitTest( pipeline, unitTest, metadataProvider, variables, results); + reported = true; + reportToLineage(pipeline, unitTest, results, null); Review Comment: **Lineage reporting can hide the unit test result.** `reportToLineage` runs inside the main validation try block, before the passed/failed logging and `showUnitTestErrors`, and it runs even when lineage is disabled. If it throws (e.g. an NPE when `getGoldenDataSets()` is null), the catch logs 'Unable to validate unit test/golden rows' and the real result is never shown. Could this go after the result handling, wrapped in its own try/catch? ########## engine/src/main/java/org/apache/hop/datacatalog/PipelineDatasets.java: ########## @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.datacatalog; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.lineage.LineageDefinitions; +import org.apache.hop.lineage.LineageFileIdentity; +import org.apache.hop.lineage.LineageRelationalIdentity; +import org.apache.hop.lineage.model.FileIoLineagePayload; +import org.apache.hop.lineage.model.FileIoOperation; +import org.apache.hop.lineage.model.LineagePayload; +import org.apache.hop.lineage.model.RelationalIoLineagePayload; +import org.apache.hop.lineage.model.RelationalTable; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.TransformMeta; + +/** + * The tables and files a pipeline declares it reads and writes, named the way lineage names them, + * so a catalog finds the datasets the lineage sinks write. Nothing runs and no database is asked + * for anything. Experimental. + */ +public final class PipelineDatasets { + + /** + * A dataset a pipeline touches. + * + * @param dataset how a catalog finds it + * @param readBy the transforms that read it + * @param writtenBy the transforms that write it + */ + public record TouchedDataset(DatasetRef dataset, Set<String> readBy, Set<String> writtenBy) {} + + private PipelineDatasets() {} + + /** + * The datasets the transforms of a pipeline declare, each once, in the order of the transforms. A + * transform whose declaration can't be resolved (a connection that doesn't exist, a variable that + * isn't set) adds nothing. + */ + public static List<TouchedDataset> of(PipelineMeta meta, IVariables variables) { + Map<DatasetRef, TouchedDataset> found = new LinkedHashMap<>(); + for (TransformMeta transform : meta.getTransforms()) { + List<LineagePayload> declared; + try { + declared = LineageDefinitions.declaredIo(meta, transform, variables, false); + } catch (Exception e) { + continue; + } + for (LineagePayload payload : declared) { + if (payload instanceof FileIoLineagePayload file) { + boolean read = file.getOperation() == FileIoOperation.READ; + LineageFileIdentity.Identity identity = + LineageFileIdentity.of(read ? file.getSourceUri() : file.getTargetUri()); + if (identity != null) { + add(found, DatasetRef.of(identity.namespace(), identity.name()), transform, read); + } + } else if (payload instanceof RelationalIoLineagePayload relational + && !Utils.isEmpty(relational.getDatasetNamespace())) { + addTables(found, relational, relational.getInputs(), transform, true); Review Comment: **Table input's tables are missing from the catalog context.** Table input declares `DeclaredSql` with empty `inputs`, because its tables are only recovered when the sink parses the SQL. So for `Table input (SELECT * FROM sales.orders) → Table output`, the AI advisor context lists only the output table. The table the pipeline mainly reads, with its description, owners and deprecation, is left out. Could the SQL parsing be shared here, so the context sees the same tables the sink does? ########## engine/src/main/java/org/apache/hop/lineage/hub/LineageHub.java: ########## @@ -375,8 +650,15 @@ private void dispatchUntilStopped() { // A sink whose classes do not load (NoClassDefFoundError) ends up here too. It used to // kill this thread silently, leaving the hub "running" but dispatching nothing. sinkInitFailed.set(true); + // None of them: some may have been initialized, the one that failed may be half-way. + sinks = List.of(); Review Comment: **A failed sink init leaks the sinks that did start.** When one of the variable-configured sinks fails, `sinks = List.of()` throws away the ones that already ran `init()` and started a dispatcher, without shutting them down. And if the worker exits for lack of targets and is restarted by a routed event, `initSinksIfNeeded()` runs the failing init again and leaks another set. Suggest shutting down the sinks that did initialize in the catch, and keeping `sinkInitFailed` so the init is not retried. ########## engine/src/main/java/org/apache/hop/lineage/LineageRunLifecycleEmitter.java: ########## @@ -512,6 +516,7 @@ public static void emitTransform( } ITransform transform = combi.transform; ctx.putAttribute("transformErrors", String.valueOf(transform.getErrors())); + LineageTransformLogic.put(ctx, combi.transformMeta, transform); Review Comment: **JSON serialisation per transform event, even with lineage disabled.** `LineageTransformLogic.put` does a Jackson `writeValueAsString` on every transform STARTED and FINISHED event, before `emit()` drops the event when lineage is off. A Pipeline Executor that runs a sub-pipeline per row pays for this twice per transform per row. Suggest checking `LineageSinkRoutes.isEnabled(transform)` first, and computing the logic only on STARTED. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
