bamaer commented on code in PR #9545:
URL: https://github.com/apache/hop/pull/9545#discussion_r4619055539


##########
plugins/tech/datahub/src/main/java/org/apache/hop/lineage/datahub/DataHubPublisher.java:
##########
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.lineage.datahub;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import lombok.Setter;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSelectInfo;
+import org.apache.commons.vfs2.FileSelector;
+import org.apache.commons.vfs2.FileType;
+import org.apache.hop.base.AbstractMeta;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.parameters.UnknownParamException;
+import org.apache.hop.core.util.Utils;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.lineage.LineageDefinitions;
+import org.apache.hop.lineage.hub.LineageSinkTarget;
+import org.apache.hop.lineage.metadata.LineageSink;
+import org.apache.hop.lineage.model.LineageEvent;
+import org.apache.hop.lineage.spi.LineageSinkHealth;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.workflow.WorkflowMeta;
+
+/**
+ * Publishes the pipelines and workflows of a folder to DataHub without 
running them: each as a flow
+ * with its transforms or actions as tasks, the tables and files they declare 
as their inputs and
+ * outputs, and the layouts of those files as dataset schemas. It writes 
through a DataHub {@link
+ * LineageSink}, so the entities are the ones the runs report to.
+ *
+ * <p>With {@code includingDocumentation} set it also publishes what {@code 
hop doc} writes for each
+ * pipeline and workflow as a DataHub document (DataHub 1.7 and later), 
related to its flow and
+ * tasks, under a document of the project.
+ */
+public final class DataHubPublisher {
+
+  /** How long to wait for DataHub to accept what was published. */
+  static final long DELIVERY_TIMEOUT_MS = 120_000L;
+
+  /** How long to wait for DataHub to take the first document, before the 
others are sent. */
+  static final long FIRST_DOCUMENT_TIMEOUT_MS = 30_000L;
+
+  /** The name of the project, set when a project is used. */
+  static final String VAR_PROJECT_NAME = "HOP_PROJECT_NAME";
+
+  /**
+   * What a publication did.
+   *
+   * @param pipelines the pipelines published
+   * @param workflows the workflows published
+   * @param unreadable the files that could not be read, and were skipped
+   * @param documentationNotPublished why the documentation was not published, 
or null
+   * @param health how the delivery to DataHub went
+   */
+  public record Result(
+      int pipelines,
+      int workflows,
+      List<String> unreadable,
+      String documentationNotPublished,
+      LineageSinkHealth health) {
+
+    /** Whether everything was read and DataHub accepted all of it. */
+    public boolean isSuccess() {
+      return unreadable.isEmpty()
+          && documentationNotPublished == null
+          && health != null
+          && health.waiting() == 0
+          && health.rejected() == 0
+          && health.dropped() == 0;
+    }
+  }
+
+  private final IVariables variables;
+  private final IHopMetadataProvider metadataProvider;
+  private final ILogChannel log;
+
+  /**
+   * Whether to publish what {@code hop doc} writes for each pipeline and 
workflow as well, as
+   * DataHub documents: DataHub 1.7 or later, an earlier version refuses them.
+   */
+  @Setter private boolean includingDocumentation;
+
+  /** Whether the documentation lists the parameters, as {@code hop doc -ip} 
does. */
+  @Setter private boolean includingParameters;
+
+  /** Whether the documentation has the text of the notes, as {@code hop doc 
-in} does. */
+  @Setter private boolean includingNotes;
+
+  public DataHubPublisher(
+      IVariables variables, IHopMetadataProvider metadataProvider, ILogChannel 
log) {
+    this.variables = variables;
+    this.metadataProvider = metadataProvider;
+    this.log = log;
+  }
+
+  /**
+   * Publishes every pipeline and workflow in {@code folder} and its 
subfolders.
+   *
+   * @param sinkName the name of a lineage sink of type DataHub
+   * @param folder the folder, variables allowed
+   * @throws HopException when the sink can't be used, or the folder doesn't 
exist
+   */
+  public Result publish(String sinkName, String folder) throws HopException {
+    DataHubLineageSink sink = startSink(sinkName);
+    int pipelines = 0;
+    int workflows = 0;
+    List<String> unreadable = new ArrayList<>();
+    List<LineageEvent> documents = new ArrayList<>();
+    String documentationNotPublished = null;
+    LineageSinkHealth health;
+    try {
+      FileObject root = folder(folder);
+      DocumentationWriter documentation = includingDocumentation ? 
documentationWriter() : null;
+      String project = variables.getVariable(VAR_PROJECT_NAME);
+      for (FileObject file : hopFiles(root)) {
+        String filename = HopVfs.getFilename(file);
+        try {
+          String relativeFile = root.getName().getRelativeName(file.getName());
+          if (filename.endsWith(".hpl")) {
+            PipelineMeta meta = new PipelineMeta(filename, metadataProvider, 
variables);
+            List<LineageEvent> events =
+                LineageDefinitions.ofPipeline(meta, variablesOf(meta, 
variables), log);
+            if (documentation != null) {
+              documents.add(documentation.pipeline(events.get(0), meta, 
relativeFile, project));

Review Comment:
   Follow-up in e88ba02b87: a file published without its documentation is now 
named in the summary, and the publication no longer counts as a success.



##########
engine/src/main/java/org/apache/hop/lineage/hub/LineageHub.java:
##########
@@ -124,12 +384,18 @@ public void environmentReady() {
     sinksInitialized.set(false);
     sinkInitFailed.set(false);
     sinks = List.of();
+    sinkIds.clear();
+    sinkInitError = null;
+    routedSinks.clear();

Review Comment:
   Follow-up in e88ba02b87: a re-init now also shuts down the sinks set with 
`HOP_LINEAGE_SINK_IDS`, which had the same leak 
(`aReInitShutsDownTheSinksOfTheVariables`).



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to