phet commented on code in PR #3890:
URL: https://github.com/apache/gobblin/pull/3890#discussion_r1525690896
##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/DagActionStore.java:
##########
@@ -40,7 +40,8 @@ class DagAction {
final String flowGroup;
final String flowName;
final String flowExecutionId;
- final FlowActionType flowActionType;
+ final String jobName;
+ final DagActionType _dagActionType;
Review Comment:
let's drop the leading `_`
##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MultiActiveLeaseArbiter.java:
##########
@@ -83,41 +87,56 @@ abstract class LeaseAttemptStatus {}
class NoLongerLeasingStatus extends LeaseAttemptStatus {}
/*
- The participant calling this method acquired the lease for the event in
question. `Flow action`'s flow execution id
+ The participant calling this method acquired the lease for the event in
question. `Dag action`'s flow execution id
is the timestamp associated with the lease and the time the caller obtained
the lease is stored within the
- `leaseAcquisitionTimestamp` field.
+ `leaseAcquisitionTimestamp` field. The `multiActiveLeaseArbiter` reference
is used to recordLeaseSuccess for the
+ current LeaseObtainedStatus via the completeLease method from a caller
without access to the {@link MultiActiveLeaseArbiter}.
*/
@Data
class LeaseObtainedStatus extends LeaseAttemptStatus {
- private final DagActionStore.DagAction flowAction;
+ private final DagActionStore.DagAction dagAction;
private final long leaseAcquisitionTimestamp;
+ @Getter
+ private final long minimumLingerDurationMillis;
+ @Getter(AccessLevel.NONE)
+ private final MultiActiveLeaseArbiter multiActiveLeaseArbiter;
/**
* @return event time in millis since epoch for the event of this lease
acquisition
*/
public long getEventTimeMillis() {
- return Long.parseLong(flowAction.getFlowExecutionId());
+ return Long.parseLong(dagAction.getFlowExecutionId());
+ }
+
+ /**
+ * Completes the lease referenced by this status object if it has not
expired.
+ * @return True if able to complete lease, False otherwise.
+ * @throws IOException
+ */
+ public boolean completeLease() throws IOException {
+ return multiActiveLeaseArbiter.recordLeaseSuccess(this);
}
}
/*
- This flow action event already has a valid lease owned by another
participant.
- `Flow action`'s flow execution id is the timestamp the lease is associated
with, however the flow action event it
+ This dag action event already has a valid lease owned by another participant.
+ `Dag action`'s flow execution id is the timestamp the lease is associated
with, however the dag action event it
corresponds to may be a different and distinct occurrence of the same event.
`minimumLingerDurationMillis` is the minimum amount of time to wait before
this participant should return to check if
the lease has completed or expired
*/
@Data
class LeasedToAnotherStatus extends LeaseAttemptStatus {
- private final DagActionStore.DagAction flowAction;
+ private final DagActionStore.DagAction dagAction;
+ @Getter
Review Comment:
again, `@Data`
##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MultiActiveLeaseArbiter.java:
##########
@@ -83,41 +87,56 @@ abstract class LeaseAttemptStatus {}
class NoLongerLeasingStatus extends LeaseAttemptStatus {}
/*
- The participant calling this method acquired the lease for the event in
question. `Flow action`'s flow execution id
+ The participant calling this method acquired the lease for the event in
question. `Dag action`'s flow execution id
is the timestamp associated with the lease and the time the caller obtained
the lease is stored within the
- `leaseAcquisitionTimestamp` field.
+ `leaseAcquisitionTimestamp` field. The `multiActiveLeaseArbiter` reference
is used to recordLeaseSuccess for the
+ current LeaseObtainedStatus via the completeLease method from a caller
without access to the {@link MultiActiveLeaseArbiter}.
*/
@Data
class LeaseObtainedStatus extends LeaseAttemptStatus {
- private final DagActionStore.DagAction flowAction;
+ private final DagActionStore.DagAction dagAction;
private final long leaseAcquisitionTimestamp;
+ @Getter
Review Comment:
no `@Getter` needed, since already `@Data`
##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/dag_action_store/MysqlDagActionStore.java:
##########
@@ -52,18 +52,19 @@ public class MysqlDagActionStore implements DagActionStore {
private final String tableName;
private final long retentionPeriodSeconds;
private String thisTableRetentionStatement;
- private static final String EXISTS_STATEMENT = "SELECT EXISTS(SELECT * FROM
%s WHERE flow_group = ? AND flow_name =? AND flow_execution_id = ? AND
dag_action = ?)";
+ private static final String EXISTS_STATEMENT = "SELECT EXISTS(SELECT * FROM
%s WHERE flow_group = ? AND flow_name = ? AND flow_execution_id = ? AND
job_name = ? AND dag_action = ?)";
- protected static final String INSERT_STATEMENT = "INSERT INTO %s
(flow_group, flow_name, flow_execution_id, dag_action) "
+ protected static final String INSERT_STATEMENT = "INSERT INTO %s
(flow_group, flow_name, flow_execution_id, job_name, dag_action) "
+ "VALUES (?, ?, ?, ?)";
Review Comment:
need to add another `?`
##########
gobblin-runtime/src/main/java/org/apache/gobblin/scheduler/SchedulerService.java:
##########
@@ -43,33 +43,40 @@
@Singleton
public class SchedulerService extends AbstractIdleService {
+ private StdSchedulerFactory schedulerFactory;
Review Comment:
again, wondering whether `org.quartz.SchedulerFactory` would do
##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/dag_action_store/MysqlDagActionStore.java:
##########
@@ -121,22 +123,34 @@ public boolean exists(String flowGroup, String flowName,
String flowExecutionId,
}
@Override
- public void addDagAction(String flowGroup, String flowName, String
flowExecutionId, FlowActionType flowActionType)
+ public boolean exists(String flowGroup, String flowName, String
flowExecutionId, DagActionType dagActionType) throws IOException, SQLException {
+ return exists(flowGroup, flowName, flowExecutionId, "", dagActionType);
+ }
+
+ @Override
+ public void addDagAction(String flowGroup, String flowName, String
flowExecutionId, String jobName, DagActionType dagActionType)
throws IOException {
dbStatementExecutor.withPreparedStatement(String.format(INSERT_STATEMENT,
tableName), insertStatement -> {
try {
int i = 0;
insertStatement.setString(++i, flowGroup);
insertStatement.setString(++i, flowName);
insertStatement.setString(++i, flowExecutionId);
- insertStatement.setString(++i, flowActionType.toString());
+ insertStatement.setString(++i, jobName);
+ insertStatement.setString(++i, dagActionType.toString());
return insertStatement.executeUpdate();
} catch (SQLException e) {
throw new IOException(String.format("Failure adding action for
DagAction: %s in table %s",
- new DagAction(flowGroup, flowName, flowExecutionId, flowActionType),
tableName), e);
+ new DagAction(flowGroup, flowName, flowExecutionId, jobName,
dagActionType), tableName), e);
}}, true);
}
+ @Override
+ public void addDagAction(String flowGroup, String flowName, String
flowExecutionId, DagActionType dagActionType)
+ throws IOException {
+ addDagAction(flowGroup, flowName, flowExecutionId, "", dagActionType);
Review Comment:
suggest a constant for "NO_JOB_NAME" or similar name, rather than direct use
of `""`
##########
gobblin-runtime/src/main/java/org/apache/gobblin/scheduler/SchedulerService.java:
##########
@@ -43,33 +43,40 @@
@Singleton
public class SchedulerService extends AbstractIdleService {
+ private StdSchedulerFactory schedulerFactory;
@Getter
private Scheduler scheduler;
private final boolean waitForJobCompletion;
private final Optional<Properties> quartzProps;
- public SchedulerService(boolean waitForJobCompletion, Optional<Properties>
quartzConfig) {
+ public SchedulerService(boolean waitForJobCompletion, Optional<Properties>
quartzConfig,
+ StdSchedulerFactory schedulerFactory) {
this.waitForJobCompletion = waitForJobCompletion;
this.quartzProps = quartzConfig;
+ if (this.schedulerFactory == null) {
+ this.schedulerFactory = new StdSchedulerFactory();
Review Comment:
I realize you may still refer to this `.impl`-package class, but that's not
such a big deal within our impl, rather than as a type within our interface
that client code sees (and may begin to depend upon)
##########
gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/DagActionStoreChangeMonitor.java:
##########
@@ -235,16 +236,16 @@ protected void processMessage(DecodeableKafkaRecord
message) {
*/
protected void handleDagAction(DagActionStore.DagAction dagAction, boolean
isStartup) {
log.info("(" + (isStartup ? "on-startup" : "post-startup") + ") DagAction
change ({}) received for flow: {}",
- dagAction.getFlowActionType(), dagAction);
- if
(dagAction.getFlowActionType().equals(DagActionStore.FlowActionType.RESUME)) {
+ dagAction.get_dagActionType(), dagAction);
+ if
(dagAction.get_dagActionType().equals(DagActionStore.DagActionType.RESUME)) {
Review Comment:
as mentioned, need to remove the `_` in the member decl
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/task/DagTask.java:
##########
@@ -40,7 +40,7 @@
@Alpha
public abstract class DagTask {
@Getter public final DagActionStore.DagAction dagAction;
- private final MultiActiveLeaseArbiter.LeaseObtainedStatus
leaseObtainedStatus;
+ @Getter private final MultiActiveLeaseArbiter.LeaseObtainedStatus
leaseObtainedStatus;
Review Comment:
no need for this. see above about `::conclude`
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/proc/LaunchDagProc.java:
##########
@@ -193,6 +193,14 @@ private void submitJobToExecutor(DagManagementStateStore
dagManagementStateStore
}
@Override
+ protected void commit(DagManagementStateStore dagManagementStateStore,
Optional<Dag<JobExecutionPlan>> dag) {
+ try {
+ this.launchDagTask.getLeaseObtainedStatus().completeLease();
Review Comment:
I believe you should implement this in `DagTask::conclude`. when you do, be
weary of the override in `LaunchDagTask`: we may not even need that
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManagementTaskStreamImpl.java:
##########
@@ -83,16 +84,14 @@ public boolean hasNext() {
@Override
public DagTask next() {
try {
- DagActionStore.DagAction dagAction = this.dagActionQueue.take();
//`take` blocks till element is not available
- // todo reconsider the use of MultiActiveLeaseArbiter
- //MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus = new
MultiActiveLeaseArbiter.LeaseObtainedStatus(dagAction);
- // todo - uncomment after flow trigger handler provides such an api
- //Properties jobProps = getJobProperties(dagAction);
- //flowTriggerHandler.getLeaseOnDagAction(jobProps, dagAction,
System.currentTimeMillis());
- //if (leaseAttemptStatus instanceof
MultiActiveLeaseArbiter.LeaseObtainedStatus) {
- // can it return null? is this iterator allowed to return null?
- return createDagTask(dagAction, new
MultiActiveLeaseArbiter.LeaseObtainedStatus(dagAction,
System.currentTimeMillis()));
- //}
+ MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus = null;
+ DagActionStore.DagAction dagAction = null;
+// while (!(leaseAttemptStatus instanceof
MultiActiveLeaseArbiter.LeaseObtainedStatus)) {
+ dagAction = this.dagActionQueue.take(); //`take` blocks till element
is not available
+// multiActiveLeaseArbiter.tryAcquireLease(dagAction);
+// }
+
+ return createDagTask(dagAction,
(MultiActiveLeaseArbiter.LeaseObtainedStatus) leaseAttemptStatus);
Review Comment:
w/o the `while` loop above (that you commented out), we can't be sure this
cast will succeed
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/proc/LaunchDagProc.java:
##########
@@ -193,6 +193,14 @@ private void submitJobToExecutor(DagManagementStateStore
dagManagementStateStore
}
@Override
+ protected void commit(DagManagementStateStore dagManagementStateStore,
Optional<Dag<JobExecutionPlan>> dag) {
Review Comment:
no need for this method. whoever has a `DagProc` also has a `DagTask` and
can operate on that directly
##########
gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java:
##########
@@ -194,7 +194,7 @@ private void initializeAppLauncherAndServices() throws
Exception {
this.jobCatalog = null;
}
- SchedulerService schedulerService = new SchedulerService(properties);
+ SchedulerService schedulerService = new SchedulerService(properties, null);
Review Comment:
rather than such widespread "low-value" changes like this, why not just
create an overloaded ctor for b/w compat? e.g.
```
@VisibleForTesting
SchedulerService(Properties props) {
this(props, null);
}
```
##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MultiActiveLeaseArbiter.java:
##########
@@ -83,41 +87,56 @@ abstract class LeaseAttemptStatus {}
class NoLongerLeasingStatus extends LeaseAttemptStatus {}
/*
- The participant calling this method acquired the lease for the event in
question. `Flow action`'s flow execution id
+ The participant calling this method acquired the lease for the event in
question. `Dag action`'s flow execution id
is the timestamp associated with the lease and the time the caller obtained
the lease is stored within the
- `leaseAcquisitionTimestamp` field.
+ `leaseAcquisitionTimestamp` field. The `multiActiveLeaseArbiter` reference
is used to recordLeaseSuccess for the
+ current LeaseObtainedStatus via the completeLease method from a caller
without access to the {@link MultiActiveLeaseArbiter}.
*/
@Data
class LeaseObtainedStatus extends LeaseAttemptStatus {
- private final DagActionStore.DagAction flowAction;
+ private final DagActionStore.DagAction dagAction;
private final long leaseAcquisitionTimestamp;
+ @Getter
+ private final long minimumLingerDurationMillis;
+ @Getter(AccessLevel.NONE)
+ private final MultiActiveLeaseArbiter multiActiveLeaseArbiter;
/**
* @return event time in millis since epoch for the event of this lease
acquisition
*/
public long getEventTimeMillis() {
- return Long.parseLong(flowAction.getFlowExecutionId());
+ return Long.parseLong(dagAction.getFlowExecutionId());
+ }
+
+ /**
+ * Completes the lease referenced by this status object if it has not
expired.
+ * @return True if able to complete lease, False otherwise.
Review Comment:
nit: capitalized `True/False` are python, but no caps in java
##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/dag_action_store/MysqlDagActionStore.java:
##########
@@ -52,18 +52,19 @@ public class MysqlDagActionStore implements DagActionStore {
private final String tableName;
private final long retentionPeriodSeconds;
private String thisTableRetentionStatement;
- private static final String EXISTS_STATEMENT = "SELECT EXISTS(SELECT * FROM
%s WHERE flow_group = ? AND flow_name =? AND flow_execution_id = ? AND
dag_action = ?)";
+ private static final String EXISTS_STATEMENT = "SELECT EXISTS(SELECT * FROM
%s WHERE flow_group = ? AND flow_name = ? AND flow_execution_id = ? AND
job_name = ? AND dag_action = ?)";
- protected static final String INSERT_STATEMENT = "INSERT INTO %s
(flow_group, flow_name, flow_execution_id, dag_action) "
+ protected static final String INSERT_STATEMENT = "INSERT INTO %s
(flow_group, flow_name, flow_execution_id, job_name, dag_action) "
+ "VALUES (?, ?, ?, ?)";
Review Comment:
need to add another `?`
##########
gobblin-runtime/src/test/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiterTest.java:
##########
@@ -224,7 +225,7 @@ public void
testConditionallyAcquireLeaseIfFinishedLeasingStatement()
DagActionStore.DagAction updatedResumeDagAction =
resumeDagAction.updateFlowExecutionId(
selectInfoResult.getEventTimeMillis());
boolean markedSuccess =
mysqlMultiActiveLeaseArbiter.recordLeaseSuccess(new LeaseObtainedStatus(
- updatedResumeDagAction,
selectInfoResult.getLeaseAcquisitionTimeMillis().get()));
+ updatedResumeDagAction,
selectInfoResult.getLeaseAcquisitionTimeMillis().get(), LINGER,null));
Review Comment:
space before `null`
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagActionReminderScheduler.java:
##########
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import org.quartz.JobDetail;
+import org.quartz.Scheduler;
+import org.quartz.SchedulerException;
+import org.quartz.Trigger;
+import org.quartz.impl.StdSchedulerFactory;
+
+import javax.inject.Inject;
+
+import org.apache.gobblin.runtime.api.DagActionStore;
+
+
+/**
+ * This class is used to keep track of reminders of pending flow action events
to execute. A host calls the
+ * {#scheduleReminderJob} on a flow action that it failed to acquire a lease
on but has not yet completed. The reminder
+ * will fire once the previous lease owner's lease is expected to expire.
+ */
+public class DagActionReminderScheduler {
+ private final Scheduler quartzScheduler;
+ private final DagManagement dagManagement;
+
+ @Inject
+ public DagActionReminderScheduler(StdSchedulerFactory schedulerFactory,
DagManagement dagManagement)
Review Comment:
couldn't the type be simply `SchedulerFactory`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]