phet commented on code in PR #3890:
URL: https://github.com/apache/gobblin/pull/3890#discussion_r1525690896


##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/DagActionStore.java:
##########
@@ -40,7 +40,8 @@ class DagAction {
     final String flowGroup;
     final String flowName;
     final String flowExecutionId;
-    final FlowActionType flowActionType;
+    final String jobName;
+    final DagActionType _dagActionType;

Review Comment:
   let's drop the leading `_`



##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MultiActiveLeaseArbiter.java:
##########
@@ -83,41 +87,56 @@ abstract class LeaseAttemptStatus {}
   class NoLongerLeasingStatus extends LeaseAttemptStatus {}
 
   /*
-  The participant calling this method acquired the lease for the event in 
question. `Flow action`'s flow execution id
+  The participant calling this method acquired the lease for the event in 
question. `Dag action`'s flow execution id
   is the timestamp associated with the lease and the time the caller obtained 
the lease is stored within the
-  `leaseAcquisitionTimestamp` field.
+  `leaseAcquisitionTimestamp` field. The `multiActiveLeaseArbiter` reference 
is used to recordLeaseSuccess for the
+  current LeaseObtainedStatus via the completeLease method from a caller 
without access to the {@link MultiActiveLeaseArbiter}.
   */
   @Data
   class LeaseObtainedStatus extends LeaseAttemptStatus {
-    private final DagActionStore.DagAction flowAction;
+    private final DagActionStore.DagAction dagAction;
     private final long leaseAcquisitionTimestamp;
+    @Getter
+    private final long minimumLingerDurationMillis;
+    @Getter(AccessLevel.NONE)
+    private final MultiActiveLeaseArbiter multiActiveLeaseArbiter;
 
     /**
      * @return event time in millis since epoch for the event of this lease 
acquisition
      */
     public long getEventTimeMillis() {
-      return Long.parseLong(flowAction.getFlowExecutionId());
+      return Long.parseLong(dagAction.getFlowExecutionId());
+    }
+
+    /**
+     * Completes the lease referenced by this status object if it has not 
expired.
+     * @return True if able to complete lease, False otherwise.
+     * @throws IOException
+     */
+    public boolean completeLease() throws IOException {
+      return multiActiveLeaseArbiter.recordLeaseSuccess(this);
     }
   }
 
   /*
-  This flow action event already has a valid lease owned by another 
participant.
-  `Flow action`'s flow execution id is the timestamp the lease is associated 
with, however the flow action event it
+  This dag action event already has a valid lease owned by another participant.
+  `Dag action`'s flow execution id is the timestamp the lease is associated 
with, however the dag action event it
   corresponds to may be a different and distinct occurrence of the same event.
   `minimumLingerDurationMillis` is the minimum amount of time to wait before 
this participant should return to check if
   the lease has completed or expired
    */
   @Data
   class LeasedToAnotherStatus extends LeaseAttemptStatus {
-    private final DagActionStore.DagAction flowAction;
+    private final DagActionStore.DagAction dagAction;
+    @Getter

Review Comment:
   again, `@Data`



##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MultiActiveLeaseArbiter.java:
##########
@@ -83,41 +87,56 @@ abstract class LeaseAttemptStatus {}
   class NoLongerLeasingStatus extends LeaseAttemptStatus {}
 
   /*
-  The participant calling this method acquired the lease for the event in 
question. `Flow action`'s flow execution id
+  The participant calling this method acquired the lease for the event in 
question. `Dag action`'s flow execution id
   is the timestamp associated with the lease and the time the caller obtained 
the lease is stored within the
-  `leaseAcquisitionTimestamp` field.
+  `leaseAcquisitionTimestamp` field. The `multiActiveLeaseArbiter` reference 
is used to recordLeaseSuccess for the
+  current LeaseObtainedStatus via the completeLease method from a caller 
without access to the {@link MultiActiveLeaseArbiter}.
   */
   @Data
   class LeaseObtainedStatus extends LeaseAttemptStatus {
-    private final DagActionStore.DagAction flowAction;
+    private final DagActionStore.DagAction dagAction;
     private final long leaseAcquisitionTimestamp;
+    @Getter

Review Comment:
   no `@Getter` needed, since already `@Data`



##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/dag_action_store/MysqlDagActionStore.java:
##########
@@ -52,18 +52,19 @@ public class MysqlDagActionStore implements DagActionStore {
   private final String tableName;
   private final long retentionPeriodSeconds;
   private String thisTableRetentionStatement;
-  private static final String EXISTS_STATEMENT = "SELECT EXISTS(SELECT * FROM 
%s WHERE flow_group = ? AND flow_name =? AND flow_execution_id = ? AND 
dag_action = ?)";
+  private static final String EXISTS_STATEMENT = "SELECT EXISTS(SELECT * FROM 
%s WHERE flow_group = ? AND flow_name = ? AND flow_execution_id = ? AND 
job_name = ? AND dag_action = ?)";
 
-  protected static final String INSERT_STATEMENT = "INSERT INTO %s 
(flow_group, flow_name, flow_execution_id, dag_action) "
+  protected static final String INSERT_STATEMENT = "INSERT INTO %s 
(flow_group, flow_name, flow_execution_id, job_name, dag_action) "
       + "VALUES (?, ?, ?, ?)";

Review Comment:
   need to add another `?`



##########
gobblin-runtime/src/main/java/org/apache/gobblin/scheduler/SchedulerService.java:
##########
@@ -43,33 +43,40 @@
 @Singleton
 public class SchedulerService extends AbstractIdleService {
 
+  private StdSchedulerFactory schedulerFactory;

Review Comment:
   again, wondering whether `org.quartz.SchedulerFactory` would do



##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/dag_action_store/MysqlDagActionStore.java:
##########
@@ -121,22 +123,34 @@ public boolean exists(String flowGroup, String flowName, 
String flowExecutionId,
   }
 
   @Override
-  public void addDagAction(String flowGroup, String flowName, String 
flowExecutionId, FlowActionType flowActionType)
+  public boolean exists(String flowGroup, String flowName, String 
flowExecutionId, DagActionType dagActionType) throws IOException, SQLException {
+    return exists(flowGroup, flowName, flowExecutionId, "", dagActionType);
+  }
+
+  @Override
+  public void addDagAction(String flowGroup, String flowName, String 
flowExecutionId, String jobName, DagActionType dagActionType)
       throws IOException {
     dbStatementExecutor.withPreparedStatement(String.format(INSERT_STATEMENT, 
tableName), insertStatement -> {
     try {
       int i = 0;
       insertStatement.setString(++i, flowGroup);
       insertStatement.setString(++i, flowName);
       insertStatement.setString(++i, flowExecutionId);
-      insertStatement.setString(++i, flowActionType.toString());
+      insertStatement.setString(++i, jobName);
+      insertStatement.setString(++i, dagActionType.toString());
       return insertStatement.executeUpdate();
     } catch (SQLException e) {
       throw new IOException(String.format("Failure adding action for 
DagAction: %s in table %s",
-          new DagAction(flowGroup, flowName, flowExecutionId, flowActionType), 
tableName), e);
+          new DagAction(flowGroup, flowName, flowExecutionId, jobName, 
dagActionType), tableName), e);
     }}, true);
   }
 
+  @Override
+  public void addDagAction(String flowGroup, String flowName, String 
flowExecutionId, DagActionType dagActionType)
+      throws IOException {
+    addDagAction(flowGroup, flowName, flowExecutionId, "", dagActionType);

Review Comment:
   suggest a constant for "NO_JOB_NAME" or similar name, rather than direct use 
of `""`



##########
gobblin-runtime/src/main/java/org/apache/gobblin/scheduler/SchedulerService.java:
##########
@@ -43,33 +43,40 @@
 @Singleton
 public class SchedulerService extends AbstractIdleService {
 
+  private StdSchedulerFactory schedulerFactory;
   @Getter
   private Scheduler scheduler;
   private final boolean waitForJobCompletion;
   private final Optional<Properties> quartzProps;
 
-  public SchedulerService(boolean waitForJobCompletion, Optional<Properties> 
quartzConfig) {
+  public SchedulerService(boolean waitForJobCompletion, Optional<Properties> 
quartzConfig,
+      StdSchedulerFactory schedulerFactory) {
     this.waitForJobCompletion = waitForJobCompletion;
     this.quartzProps = quartzConfig;
+    if (this.schedulerFactory == null) {
+      this.schedulerFactory = new StdSchedulerFactory();

Review Comment:
   I realize you may still refer to this `.impl`-package class, but that's not 
such a big deal within our impl, rather than as a type within our interface 
that client code sees (and may begin to depend upon)



##########
gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/DagActionStoreChangeMonitor.java:
##########
@@ -235,16 +236,16 @@ protected void processMessage(DecodeableKafkaRecord 
message) {
    */
   protected void handleDagAction(DagActionStore.DagAction dagAction, boolean 
isStartup) {
     log.info("(" + (isStartup ? "on-startup" : "post-startup") + ") DagAction 
change ({}) received for flow: {}",
-        dagAction.getFlowActionType(), dagAction);
-    if 
(dagAction.getFlowActionType().equals(DagActionStore.FlowActionType.RESUME)) {
+        dagAction.get_dagActionType(), dagAction);
+    if 
(dagAction.get_dagActionType().equals(DagActionStore.DagActionType.RESUME)) {

Review Comment:
   as mentioned, need to remove the `_` in the member decl



##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/task/DagTask.java:
##########
@@ -40,7 +40,7 @@
 @Alpha
 public abstract class DagTask {
   @Getter public final DagActionStore.DagAction dagAction;
-  private final MultiActiveLeaseArbiter.LeaseObtainedStatus 
leaseObtainedStatus;
+  @Getter private final MultiActiveLeaseArbiter.LeaseObtainedStatus 
leaseObtainedStatus;

Review Comment:
   no need for this.  see above about `::conclude`



##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/proc/LaunchDagProc.java:
##########
@@ -193,6 +193,14 @@ private void submitJobToExecutor(DagManagementStateStore 
dagManagementStateStore
   }
 
   @Override
+  protected void commit(DagManagementStateStore dagManagementStateStore, 
Optional<Dag<JobExecutionPlan>> dag) {
+    try {
+      this.launchDagTask.getLeaseObtainedStatus().completeLease();

Review Comment:
   I believe you should implement this in `DagTask::conclude`.  when you do, be 
weary of the override in `LaunchDagTask`: we may not even need that



##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManagementTaskStreamImpl.java:
##########
@@ -83,16 +84,14 @@ public boolean hasNext() {
   @Override
   public DagTask next() {
     try {
-      DagActionStore.DagAction dagAction = this.dagActionQueue.take();  
//`take` blocks till element is not available
-      // todo reconsider the use of MultiActiveLeaseArbiter
-      //MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus = new 
MultiActiveLeaseArbiter.LeaseObtainedStatus(dagAction);
-      // todo - uncomment after flow trigger handler provides such an api
-      //Properties jobProps = getJobProperties(dagAction);
-      //flowTriggerHandler.getLeaseOnDagAction(jobProps, dagAction, 
System.currentTimeMillis());
-      //if (leaseAttemptStatus instanceof 
MultiActiveLeaseArbiter.LeaseObtainedStatus) {
-      // can it return null? is this iterator allowed to return null?
-      return createDagTask(dagAction, new 
MultiActiveLeaseArbiter.LeaseObtainedStatus(dagAction, 
System.currentTimeMillis()));
-      //}
+      MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus = null;
+      DagActionStore.DagAction dagAction = null;
+//      while (!(leaseAttemptStatus instanceof 
MultiActiveLeaseArbiter.LeaseObtainedStatus)) {
+        dagAction = this.dagActionQueue.take();  //`take` blocks till element 
is not available
+//        multiActiveLeaseArbiter.tryAcquireLease(dagAction);
+//      }
+
+      return createDagTask(dagAction, 
(MultiActiveLeaseArbiter.LeaseObtainedStatus) leaseAttemptStatus);

Review Comment:
   w/o the `while` loop above (that you commented out), we can't be sure this 
cast will succeed



##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/proc/LaunchDagProc.java:
##########
@@ -193,6 +193,14 @@ private void submitJobToExecutor(DagManagementStateStore 
dagManagementStateStore
   }
 
   @Override
+  protected void commit(DagManagementStateStore dagManagementStateStore, 
Optional<Dag<JobExecutionPlan>> dag) {

Review Comment:
   no need for this method.  whoever has a `DagProc` also has a `DagTask` and 
can operate on that directly



##########
gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java:
##########
@@ -194,7 +194,7 @@ private void initializeAppLauncherAndServices() throws 
Exception {
       this.jobCatalog = null;
     }
 
-    SchedulerService schedulerService = new SchedulerService(properties);
+    SchedulerService schedulerService = new SchedulerService(properties, null);

Review Comment:
   rather than such widespread "low-value" changes like this, why not just 
create an overloaded ctor for b/w compat?  e.g.
   ```
   @VisibleForTesting
   SchedulerService(Properties props) {
     this(props, null);
   }
   ```



##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MultiActiveLeaseArbiter.java:
##########
@@ -83,41 +87,56 @@ abstract class LeaseAttemptStatus {}
   class NoLongerLeasingStatus extends LeaseAttemptStatus {}
 
   /*
-  The participant calling this method acquired the lease for the event in 
question. `Flow action`'s flow execution id
+  The participant calling this method acquired the lease for the event in 
question. `Dag action`'s flow execution id
   is the timestamp associated with the lease and the time the caller obtained 
the lease is stored within the
-  `leaseAcquisitionTimestamp` field.
+  `leaseAcquisitionTimestamp` field. The `multiActiveLeaseArbiter` reference 
is used to recordLeaseSuccess for the
+  current LeaseObtainedStatus via the completeLease method from a caller 
without access to the {@link MultiActiveLeaseArbiter}.
   */
   @Data
   class LeaseObtainedStatus extends LeaseAttemptStatus {
-    private final DagActionStore.DagAction flowAction;
+    private final DagActionStore.DagAction dagAction;
     private final long leaseAcquisitionTimestamp;
+    @Getter
+    private final long minimumLingerDurationMillis;
+    @Getter(AccessLevel.NONE)
+    private final MultiActiveLeaseArbiter multiActiveLeaseArbiter;
 
     /**
      * @return event time in millis since epoch for the event of this lease 
acquisition
      */
     public long getEventTimeMillis() {
-      return Long.parseLong(flowAction.getFlowExecutionId());
+      return Long.parseLong(dagAction.getFlowExecutionId());
+    }
+
+    /**
+     * Completes the lease referenced by this status object if it has not 
expired.
+     * @return True if able to complete lease, False otherwise.

Review Comment:
   nit: capitalized `True/False` are python, but no caps in java



##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/dag_action_store/MysqlDagActionStore.java:
##########
@@ -52,18 +52,19 @@ public class MysqlDagActionStore implements DagActionStore {
   private final String tableName;
   private final long retentionPeriodSeconds;
   private String thisTableRetentionStatement;
-  private static final String EXISTS_STATEMENT = "SELECT EXISTS(SELECT * FROM 
%s WHERE flow_group = ? AND flow_name =? AND flow_execution_id = ? AND 
dag_action = ?)";
+  private static final String EXISTS_STATEMENT = "SELECT EXISTS(SELECT * FROM 
%s WHERE flow_group = ? AND flow_name = ? AND flow_execution_id = ? AND 
job_name = ? AND dag_action = ?)";
 
-  protected static final String INSERT_STATEMENT = "INSERT INTO %s 
(flow_group, flow_name, flow_execution_id, dag_action) "
+  protected static final String INSERT_STATEMENT = "INSERT INTO %s 
(flow_group, flow_name, flow_execution_id, job_name, dag_action) "
       + "VALUES (?, ?, ?, ?)";

Review Comment:
   need to add another `?`



##########
gobblin-runtime/src/test/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiterTest.java:
##########
@@ -224,7 +225,7 @@ public void 
testConditionallyAcquireLeaseIfFinishedLeasingStatement()
     DagActionStore.DagAction updatedResumeDagAction = 
resumeDagAction.updateFlowExecutionId(
         selectInfoResult.getEventTimeMillis());
     boolean markedSuccess = 
mysqlMultiActiveLeaseArbiter.recordLeaseSuccess(new LeaseObtainedStatus(
-        updatedResumeDagAction, 
selectInfoResult.getLeaseAcquisitionTimeMillis().get()));
+        updatedResumeDagAction, 
selectInfoResult.getLeaseAcquisitionTimeMillis().get(), LINGER,null));

Review Comment:
   space before `null`



##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagActionReminderScheduler.java:
##########
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import org.quartz.JobDetail;
+import org.quartz.Scheduler;
+import org.quartz.SchedulerException;
+import org.quartz.Trigger;
+import org.quartz.impl.StdSchedulerFactory;
+
+import javax.inject.Inject;
+
+import org.apache.gobblin.runtime.api.DagActionStore;
+
+
+/**
+ * This class is used to keep track of reminders of pending flow action events 
to execute. A host calls the
+ * {#scheduleReminderJob} on a flow action that it failed to acquire a lease 
on but has not yet completed. The reminder
+ * will fire once the previous lease owner's lease is expected to expire.
+ */
+public class DagActionReminderScheduler {
+  private final Scheduler quartzScheduler;
+  private final DagManagement dagManagement;
+
+  @Inject
+  public DagActionReminderScheduler(StdSchedulerFactory schedulerFactory, 
DagManagement dagManagement)

Review Comment:
   couldn't the type be simply `SchedulerFactory`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to