bamaer commented on code in PR #9545: URL: https://github.com/apache/hop/pull/9545#discussion_r4619028612
########## plugins/tech/datahub/src/main/java/org/apache/hop/lineage/datahub/DataHubPublisher.java: ########## @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.lineage.datahub; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import lombok.Setter; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSelectInfo; +import org.apache.commons.vfs2.FileSelector; +import org.apache.commons.vfs2.FileType; +import org.apache.hop.base.AbstractMeta; +import org.apache.hop.core.Const; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.logging.ILogChannel; +import org.apache.hop.core.parameters.UnknownParamException; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.lineage.LineageDefinitions; +import org.apache.hop.lineage.hub.LineageSinkTarget; +import org.apache.hop.lineage.metadata.LineageSink; +import org.apache.hop.lineage.model.LineageEvent; +import org.apache.hop.lineage.spi.LineageSinkHealth; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.workflow.WorkflowMeta; + +/** + * Publishes the pipelines and workflows of a folder to DataHub without running them: each as a flow + * with its transforms or actions as tasks, the tables and files they declare as their inputs and + * outputs, and the layouts of those files as dataset schemas. It writes through a DataHub {@link + * LineageSink}, so the entities are the ones the runs report to. + * + * <p>With {@code includingDocumentation} set it also publishes what {@code hop doc} writes for each + * pipeline and workflow as a DataHub document (DataHub 1.7 and later), related to its flow and + * tasks, under a document of the project. + */ +public final class DataHubPublisher { + + /** How long to wait for DataHub to accept what was published. */ + static final long DELIVERY_TIMEOUT_MS = 120_000L; + + /** How long to wait for DataHub to take the first document, before the others are sent. */ + static final long FIRST_DOCUMENT_TIMEOUT_MS = 30_000L; + + /** The name of the project, set when a project is used. */ + static final String VAR_PROJECT_NAME = "HOP_PROJECT_NAME"; + + /** + * What a publication did. + * + * @param pipelines the pipelines published + * @param workflows the workflows published + * @param unreadable the files that could not be read, and were skipped + * @param documentationNotPublished why the documentation was not published, or null + * @param health how the delivery to DataHub went + */ + public record Result( + int pipelines, + int workflows, + List<String> unreadable, + String documentationNotPublished, + LineageSinkHealth health) { + + /** Whether everything was read and DataHub accepted all of it. */ + public boolean isSuccess() { + return unreadable.isEmpty() + && documentationNotPublished == null + && health != null + && health.waiting() == 0 + && health.rejected() == 0 + && health.dropped() == 0; + } + } + + private final IVariables variables; + private final IHopMetadataProvider metadataProvider; + private final ILogChannel log; + + /** + * Whether to publish what {@code hop doc} writes for each pipeline and workflow as well, as + * DataHub documents: DataHub 1.7 or later, an earlier version refuses them. + */ + @Setter private boolean includingDocumentation; + + /** Whether the documentation lists the parameters, as {@code hop doc -ip} does. */ + @Setter private boolean includingParameters; + + /** Whether the documentation has the text of the notes, as {@code hop doc -in} does. */ + @Setter private boolean includingNotes; + + public DataHubPublisher( + IVariables variables, IHopMetadataProvider metadataProvider, ILogChannel log) { + this.variables = variables; + this.metadataProvider = metadataProvider; + this.log = log; + } + + /** + * Publishes every pipeline and workflow in {@code folder} and its subfolders. + * + * @param sinkName the name of a lineage sink of type DataHub + * @param folder the folder, variables allowed + * @throws HopException when the sink can't be used, or the folder doesn't exist + */ + public Result publish(String sinkName, String folder) throws HopException { + DataHubLineageSink sink = startSink(sinkName); + int pipelines = 0; + int workflows = 0; + List<String> unreadable = new ArrayList<>(); + List<LineageEvent> documents = new ArrayList<>(); + String documentationNotPublished = null; + LineageSinkHealth health; + try { + FileObject root = folder(folder); + DocumentationWriter documentation = includingDocumentation ? documentationWriter() : null; + String project = variables.getVariable(VAR_PROJECT_NAME); + for (FileObject file : hopFiles(root)) { + String filename = HopVfs.getFilename(file); + try { + String relativeFile = root.getName().getRelativeName(file.getName()); + if (filename.endsWith(".hpl")) { + PipelineMeta meta = new PipelineMeta(filename, metadataProvider, variables); + List<LineageEvent> events = + LineageDefinitions.ofPipeline(meta, variablesOf(meta, variables), log); + if (documentation != null) { + documents.add(documentation.pipeline(events.get(0), meta, relativeFile, project)); Review Comment: Done in b82b2aa1bc: the events are sent first, and the documentation has its own try/catch, for pipelines and workflows. A file whose document can't be written keeps its lineage; only the document is left out, with an error in the log (`aPipelineWhoseDocumentationFailsIsStillPublished`). ########## plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/ValidatePipelineUnitTestExtensionPoint.java: ########## @@ -73,17 +81,17 @@ public void callExtensionPoint( // If the pipeline has a variable set with the unit test in it, we're dealing with a unit test // situation. // - PipelineUnitTest unitTest = - metadataProvider.getSerializer(PipelineUnitTest.class).load(unitTestName); + unitTest = metadataProvider.getSerializer(PipelineUnitTest.class).load(unitTestName); - final List<UnitTestResult> results = new ArrayList<>(); pipeline.getExtensionDataMap().put(DataSetConst.UNIT_TEST_RESULTS, results); // Validate execution results with what's in the data sets... // int errors = DataSetConst.validateTransformResultAgainstUnitTest( pipeline, unitTest, metadataProvider, variables, results); + reported = true; + reportToLineage(pipeline, unitTest, results, null); Review Comment: Done: the lineage report now runs after the result is logged and shown, in its own try/catch, and only when the run has lineage sinks. A test without golden data sets is handled too (`aUnitTestWithoutGoldenDataSetsReportsOnlyItsResults`). ########## plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/textfileoutput/TextFileOutputMeta.java: ########## @@ -683,6 +689,86 @@ String[] getFiles( } } + /** + * The layout of the file this transform writes: the output fields when some are listed, the whole + * row otherwise. + * + * @param outputRowMeta the row the transform writes; null or empty when it is not known, then + * only listed fields give a layout, with the types they are listed with + */ + public FileIoContentSchema writtenContentSchema(IRowMeta outputRowMeta) { + List<FileIoTabularColumn> cols = new ArrayList<>(); + if (outputFields != null && !outputFields.isEmpty()) { + for (TextFileField f : outputFields) { + String typeDesc = f.getTypeDesc(); + if (outputRowMeta != null) { + IValueMeta v = outputRowMeta.searchValueMeta(f.getName()); + if (v != null) { + typeDesc = v.getTypeDesc(); + } + } + cols.add( + new FileIoTabularColumn( + f.getName(), + typeDesc, + f.getLength(), + f.getPrecision(), + null, + FileIoPathSyntax.NONE, + false)); + } + } else if (outputRowMeta != null) { + for (int i = 0; i < outputRowMeta.size(); i++) { + IValueMeta v = outputRowMeta.getValueMeta(i); + cols.add( + new FileIoTabularColumn( + v.getName(), + v.getTypeDesc(), + v.getLength(), + v.getPrecision(), + null, + FileIoPathSyntax.DELIMITED, + false)); + } + } + if (cols.isEmpty()) { + return null; + } + return new FileIoContentSchema("delimited", cols, List.of()); + } + + /** + * The file a run writes first, with its layout; none when the name comes from a field, or holds + * the date or time of the run, so no publication names a file no run writes. + */ + @Override + public List<IDeclaresLineage.Declaration> declaredLineage( + IVariables variables, IRowMeta inputRowMeta) { + boolean dated = + (fileSettings.isSpecifyingFormat() && !Utils.isEmpty(fileSettings.getDateTimeFormat())) + || (!fileSettings.isSpecifyingFormat() + && (fileSettings.isDateInFilename() || fileSettings.isTimeInFilename())); + if (fileNameInField || Utils.isEmpty(fileSettings.fileName) || dated) { + return List.of(); + } + String filename = + buildFilename( Review Comment: Done: with the partition number in the name, no file is declared, the same as for a name with the date or time. I went for nothing rather than the base name, because no run writes the base name either; the run-time lineage still reports the real files. Date and time were already left out in the commit you reviewed (the `dated` check), with a test. ########## engine/src/main/java/org/apache/hop/lineage/hub/LineageHub.java: ########## @@ -124,12 +384,18 @@ public void environmentReady() { sinksInitialized.set(false); sinkInitFailed.set(false); sinks = List.of(); + sinkIds.clear(); + sinkInitError = null; + routedSinks.clear(); Review Comment: Done: `environmentReady()` shuts the routed sinks down in the background before it clears the map (`aReInitShutsDownTheSinksRunsStarted`). ########## engine/src/main/java/org/apache/hop/lineage/hub/LineageHub.java: ########## @@ -375,8 +650,15 @@ private void dispatchUntilStopped() { // A sink whose classes do not load (NoClassDefFoundError) ends up here too. It used to // kill this thread silently, leaving the hub "running" but dispatching nothing. sinkInitFailed.set(true); + // None of them: some may have been initialized, the one that failed may be half-way. + sinks = List.of(); Review Comment: Done: when one sink fails, every sink `init()` was called on is shut down, the failing one included, and the init isn't tried again until `shutdown()` or `environmentReady()` (`aSinkThatFailsToInitShutsDownTheOnesThatStartedAndIsNotTriedAgain`). ########## engine/src/main/java/org/apache/hop/datacatalog/PipelineDatasets.java: ########## @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.datacatalog; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.lineage.LineageDefinitions; +import org.apache.hop.lineage.LineageFileIdentity; +import org.apache.hop.lineage.LineageRelationalIdentity; +import org.apache.hop.lineage.model.FileIoLineagePayload; +import org.apache.hop.lineage.model.FileIoOperation; +import org.apache.hop.lineage.model.LineagePayload; +import org.apache.hop.lineage.model.RelationalIoLineagePayload; +import org.apache.hop.lineage.model.RelationalTable; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.TransformMeta; + +/** + * The tables and files a pipeline declares it reads and writes, named the way lineage names them, + * so a catalog finds the datasets the lineage sinks write. Nothing runs and no database is asked + * for anything. Experimental. + */ +public final class PipelineDatasets { + + /** + * A dataset a pipeline touches. + * + * @param dataset how a catalog finds it + * @param readBy the transforms that read it + * @param writtenBy the transforms that write it + */ + public record TouchedDataset(DatasetRef dataset, Set<String> readBy, Set<String> writtenBy) {} + + private PipelineDatasets() {} + + /** + * The datasets the transforms of a pipeline declare, each once, in the order of the transforms. A + * transform whose declaration can't be resolved (a connection that doesn't exist, a variable that + * isn't set) adds nothing. + */ + public static List<TouchedDataset> of(PipelineMeta meta, IVariables variables) { + Map<DatasetRef, TouchedDataset> found = new LinkedHashMap<>(); + for (TransformMeta transform : meta.getTransforms()) { + List<LineagePayload> declared; + try { + declared = LineageDefinitions.declaredIo(meta, transform, variables, false); + } catch (Exception e) { + continue; + } + for (LineagePayload payload : declared) { + if (payload instanceof FileIoLineagePayload file) { + boolean read = file.getOperation() == FileIoOperation.READ; + LineageFileIdentity.Identity identity = + LineageFileIdentity.of(read ? file.getSourceUri() : file.getTargetUri()); + if (identity != null) { + add(found, DatasetRef.of(identity.namespace(), identity.name()), transform, read); + } + } else if (payload instanceof RelationalIoLineagePayload relational + && !Utils.isEmpty(relational.getDatasetNamespace())) { + addTables(found, relational, relational.getInputs(), transform, true); Review Comment: Done: the engine has a small `LineageSqlTables` hook, and the OpenLineage plugin registers its SQL parser there at `HopEnvironmentAfterInit`. `PipelineDatasets` uses it when a payload only carries SQL, so the context gets the same tables as the sinks. Without the OpenLineage plugin those tables stay unknown; the AI advisor page says so. ########## engine/src/main/java/org/apache/hop/lineage/LineageRunLifecycleEmitter.java: ########## @@ -512,6 +516,7 @@ public static void emitTransform( } ITransform transform = combi.transform; ctx.putAttribute("transformErrors", String.valueOf(transform.getErrors())); + LineageTransformLogic.put(ctx, combi.transformMeta, transform); Review Comment: Done: the logic is only computed on STARTED, and only when `LineageSinkRoutes.isEnabled(transform)`. DataHub reads it from the start event and OpenLineage merges the job facets of a run's events, so nothing is lost on FINISHED. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
