[ 
https://issues.apache.org/jira/browse/GOBBLIN-1989?focusedWorklogId=901102&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-901102
 ]

ASF GitHub Bot logged work on GOBBLIN-1989:
-------------------------------------------

                Author: ASF GitHub Bot
            Created on: 23/Jan/24 03:55
            Start Date: 23/Jan/24 03:55
    Worklog Time Spent: 10m 
      Work Description: phet commented on code in PR #3843:
URL: https://github.com/apache/gobblin/pull/3843#discussion_r1462671637


##########
gobblin-temporal/src/main/java/org/apache/gobblin/temporal/ddm/workflow/impl/ProcessWorkUnitsWorkflowImpl.java:
##########
@@ -16,35 +16,86 @@
  */
 package org.apache.gobblin.temporal.ddm.workflow.impl;
 
+import java.io.IOException;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
+import java.util.function.Consumer;
+
+import org.apache.hadoop.fs.FileSystem;
 
 import com.typesafe.config.ConfigFactory;
 
+import io.temporal.activity.ActivityOptions;
 import io.temporal.api.enums.v1.ParentClosePolicy;
 import io.temporal.workflow.ChildWorkflowOptions;
 import io.temporal.workflow.Workflow;
 import lombok.extern.slf4j.Slf4j;
 
+import org.apache.gobblin.instrumented.GobblinMetricsKeys;
+import org.apache.gobblin.metrics.GobblinMetrics;
+import org.apache.gobblin.metrics.Tag;
+import org.apache.gobblin.metrics.event.TimingEvent;
+import org.apache.gobblin.runtime.JobState;
+import org.apache.gobblin.runtime.util.JobMetrics;
 import org.apache.gobblin.temporal.cluster.WorkerConfig;
+import 
org.apache.gobblin.temporal.ddm.work.EagerFsDirBackedWorkUnitClaimCheckWorkload;
 import org.apache.gobblin.temporal.ddm.work.WUProcessingSpec;
 import org.apache.gobblin.temporal.ddm.work.WorkUnitClaimCheck;
 import org.apache.gobblin.temporal.ddm.work.assistance.Help;
 import org.apache.gobblin.temporal.ddm.work.styles.FileSystemJobStateful;
 import org.apache.gobblin.temporal.ddm.workflow.CommitStepWorkflow;
 import org.apache.gobblin.temporal.ddm.workflow.ProcessWorkUnitsWorkflow;
-import 
org.apache.gobblin.temporal.ddm.work.EagerFsDirBackedWorkUnitClaimCheckWorkload;
 import org.apache.gobblin.temporal.util.nesting.work.WorkflowAddr;
 import org.apache.gobblin.temporal.util.nesting.work.Workload;
 import org.apache.gobblin.temporal.util.nesting.workflow.NestingExecWorkflow;
+import org.apache.gobblin.temporal.workflows.metrics.EventTimer;
+import org.apache.gobblin.temporal.workflows.metrics.SubmitGTEActivity;
+import org.apache.gobblin.temporal.workflows.metrics.TemporalEventTimer;
+import org.apache.gobblin.temporal.workflows.metrics.TrackingEventMetadata;
 
 
 @Slf4j
 public class ProcessWorkUnitsWorkflowImpl implements ProcessWorkUnitsWorkflow {
   public static final String CHILD_WORKFLOW_ID_BASE = "NestingExecWorkUnits";
   public static final String COMMIT_STEP_WORKFLOW_ID_BASE = 
"CommitStepWorkflow";
+  private final ActivityOptions options = ActivityOptions.newBuilder()
+      .setStartToCloseTimeout(Duration.ofSeconds(60))
+      .build();
+  private final SubmitGTEActivity submitGTEActivity = Workflow.newActivityStub(
+      SubmitGTEActivity.class, options);
+
+  private static final int MAX_DESERIALIZATION_FS_LOAD_ATTEMPTS = 5;
 
   @Override
   public int process(WUProcessingSpec workSpec) {
+    // NOTE: We are using the metrics tags from Job Props to create the metric 
context for the timer and NOT
+    // the deserialized jobState from HDFS that is created by the real distcp 
job. This is because the AZ runtime
+    // settings we want are for the job launcher that launched this Yarn job.
+
+    try {
+      FileSystem fs = Help.loadFileSystemForce(workSpec);
+      JobState jobState = Help.loadJobStateUncached(workSpec, fs);
+      List<Tag<?>> tagsFromCurrentJob = workSpec.getTags();
+      String metricsSuffix = workSpec.getMetricsSuffix();
+      List<Tag<?>> tags = getTags(tagsFromCurrentJob, metricsSuffix, jobState);
+
+      TemporalEventTimer.Factory timerFactory = new 
TemporalEventTimer.Factory(submitGTEActivity, new TrackingEventMetadata(tags, 
JobMetrics.NAMESPACE));
+      try (EventTimer timer = timerFactory.getJobTimer()) {
+        return performWork(workSpec);

Review Comment:
   don't we need a way to pass info from `performWork()` into the GTE that's 
submitted/emitted?
   
   e.g. we want to indicate whether the completion status is FAILED, CANCELED, 
COMPLETE?  similarly, the GTE could convey the total size of successful copy 
entities.
   
   for that would we need a method like `EventTimer::addMetadata` that could be 
called on `timer` (whether in the main flow or a `catch` block)?





Issue Time Tracking
-------------------

    Worklog Id:     (was: 901102)
    Time Spent: 40m  (was: 0.5h)

> TemporaI: Implementation of GTE for GaaS Observability Event in MR 
> deprecation for distcp
> -----------------------------------------------------------------------------------------
>
>                 Key: GOBBLIN-1989
>                 URL: https://issues.apache.org/jira/browse/GOBBLIN-1989
>             Project: Apache Gobblin
>          Issue Type: Improvement
>            Reporter: Matthew Ho
>            Priority: Major
>          Time Spent: 40m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian Jira
(v8.20.10#820010)

Reply via email to