phet commented on code in PR #3890:
URL: https://github.com/apache/gobblin/pull/3890#discussion_r1525733934
##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiter.java:
##########
@@ -237,19 +239,19 @@ private void initializeConstantsTable() throws
IOException {
}
@Override
- public LeaseAttemptStatus tryAcquireLease(DagActionStore.DagAction
flowAction, long eventTimeMillis,
+ public LeaseAttemptStatus tryAcquireLease(DagActionStore.DagAction
dagAction, long eventTimeMillis,
boolean isReminderEvent, boolean adoptConsensusFlowExecutionId) throws
IOException {
log.info("Multi-active scheduler about to handle trigger event: [{}, is:
{}, triggerEventTimestamp: {}]",
- flowAction, isReminderEvent ? "reminder" : "original",
eventTimeMillis);
- // Query lease arbiter table about this flow action
- Optional<GetEventInfoResult> getResult = getExistingEventInfo(flowAction,
isReminderEvent, eventTimeMillis);
+ dagAction, isReminderEvent ? "reminder" : "original", eventTimeMillis);
+ // Query lease arbiter table about this dag action
+ Optional<GetEventInfoResult> getResult = getExistingEventInfo(dagAction,
isReminderEvent, eventTimeMillis);
try {
if (!getResult.isPresent()) {
- log.debug("tryAcquireLease for [{}, is; {}, eventTimestamp: {}] - CASE
1: no existing row for this flow action,"
- + " then go ahead and insert", flowAction, isReminderEvent ?
"reminder" : "original", eventTimeMillis);
- int numRowsUpdated = attemptLeaseIfNewRow(flowAction);
- return evaluateStatusAfterLeaseAttempt(numRowsUpdated, flowAction,
Optional.empty(),
+ log.debug("tryAcquireLease for [{}, is; {}, eventTimestamp: {}] - CASE
1: no existing row for this dag action,"
+ + " then go ahead and insert", dagAction, isReminderEvent ?
"reminder" : "original", eventTimeMillis);
Review Comment:
one thing I appreciate about using the `dagAction.toString()` abstraction
for logging is that we needn't update anything about `dagAction.getJobName()`!
##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiter.java:
##########
@@ -155,8 +156,8 @@ public class MysqlMultiActiveLeaseArbiter implements
MultiActiveLeaseArbiter {
+ WHERE_CLAUSE_TO_MATCH_KEY;
// Insert or update row to acquire lease if values have not changed since
the previous read
// Need to define three separate statements to handle cases where row does
not exist or has null values to check
- protected static final String ACQUIRE_LEASE_IF_NEW_ROW_STATEMENT = "INSERT
INTO %s (flow_group, flow_name, "
- + "flow_action, event_timestamp, lease_acquisition_timestamp) VALUES(?,
?, ?, CURRENT_TIMESTAMP(3), "
+ protected static final String ACQUIRE_LEASE_IF_NEW_ROW_STATEMENT = "INSERT
INTO %s (flow_group, flow_name, job_name"
+ + "dag_action, event_timestamp, lease_acquisition_timestamp) VALUES(?,
?, ?, CURRENT_TIMESTAMP(3), "
Review Comment:
also need another `?` here
##########
gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiter.java:
##########
@@ -489,78 +492,81 @@ protected static SelectInfoResult
createSelectInfoResult(ResultSet resultSet) th
* @throws IOException
*/
protected LeaseAttemptStatus evaluateStatusAfterLeaseAttempt(int
numRowsUpdated,
- DagActionStore.DagAction flowAction, Optional<Timestamp>
dbCurrentTimestamp, boolean isReminderEvent,
+ DagActionStore.DagAction dagAction, Optional<Timestamp>
dbCurrentTimestamp, boolean isReminderEvent,
boolean adoptConsensusFlowExecutionId)
throws SQLException, IOException {
// Fetch values in row after attempted insert
- SelectInfoResult selectInfoResult = getRowInfo(flowAction);
+ SelectInfoResult selectInfoResult = getRowInfo(dagAction);
// Another participant won the lease in between
if (!selectInfoResult.getLeaseAcquisitionTimeMillis().isPresent()) {
return new NoLongerLeasingStatus();
}
- DagActionStore.DagAction updatedFlowAction =
- adoptConsensusFlowExecutionId ?
flowAction.updateFlowExecutionId(selectInfoResult.eventTimeMillis) : flowAction;
+ DagActionStore.DagAction updatedDagAction =
+ adoptConsensusFlowExecutionId ?
dagAction.updateFlowExecutionId(selectInfoResult.eventTimeMillis) : dagAction;
+ long minimumLingerDurationMillis =
selectInfoResult.getLeaseAcquisitionTimeMillis().get() +
selectInfoResult.getDbLinger()
+ - (dbCurrentTimestamp.isPresent() ? dbCurrentTimestamp.get().getTime()
: System.currentTimeMillis());
Review Comment:
likely not major, but strictly speaking, best would be to use the DB
function `CURRENT_TIMESTAMP`. add that to the query returning these results.
otherwise we're using the DB time for most things, but still mixing in the
host's own system time here
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/InstrumentedLeaseArbiter.java:
##########
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.io.IOException;
+
+import com.typesafe.config.Config;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.gobblin.instrumented.Instrumented;
+import org.apache.gobblin.metrics.ContextAwareCounter;
+import org.apache.gobblin.metrics.ContextAwareMeter;
+import org.apache.gobblin.metrics.MetricContext;
+import org.apache.gobblin.metrics.ServiceMetricNames;
+import org.apache.gobblin.runtime.api.DagActionStore;
+import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter;
+import org.apache.gobblin.util.ConfigUtils;
+
+
+/*
+ A generic lease arbitration decorator built upon the {@link
MultiActiveLeaseArbiter} which encapsulates common
+ functionality desired by lease arbiter users to track metrics on all lease
attempts made by the arbiter. The metrics
+ can be used to compare relative performance among arbitration participants.
Review Comment:
suggest: "... performance, such as collisions/contention, among ..."
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/InstrumentedLeaseArbiter.java:
##########
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.io.IOException;
+
+import com.typesafe.config.Config;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.gobblin.instrumented.Instrumented;
+import org.apache.gobblin.metrics.ContextAwareCounter;
+import org.apache.gobblin.metrics.ContextAwareMeter;
+import org.apache.gobblin.metrics.MetricContext;
+import org.apache.gobblin.metrics.ServiceMetricNames;
+import org.apache.gobblin.runtime.api.DagActionStore;
+import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter;
+import org.apache.gobblin.util.ConfigUtils;
+
+
+/*
+ A generic lease arbitration decorator built upon the {@link
MultiActiveLeaseArbiter} which encapsulates common
+ functionality desired by lease arbiter users to track metrics on all lease
attempts made by the arbiter. The metrics
+ can be used to compare relative performance among arbitration participants.
+ */
+@Slf4j
+public class InstrumentedLeaseArbiter implements MultiActiveLeaseArbiter {
+ protected MultiActiveLeaseArbiter decoratedMultiActiveLeaseArbiter;
+ @Getter
+ protected MetricContext metricContext;
Review Comment:
`private`? or do you expect to derive further from this decorator class?
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ReminderSettingDagProcLeaseArbiter.java:
##########
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.quartz.Job;
+import org.quartz.JobBuilder;
+import org.quartz.JobDataMap;
+import org.quartz.JobDetail;
+import org.quartz.JobExecutionContext;
+import org.quartz.SchedulerException;
+import org.quartz.Trigger;
+import org.quartz.TriggerBuilder;
+
+import com.typesafe.config.Config;
+
+import javax.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.gobblin.configuration.ConfigurationKeys;
+import org.apache.gobblin.runtime.api.DagActionStore;
+import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter;
+
+
+/**
+ * Decorator used to coordinate multiple hosts with execution components
enabled to respond to flow action events with
+ * added capabilities to properly handle the result of attempted ownership
over these flow action events. It uses the
+ * {@link MultiActiveLeaseArbiter} to determine a single lease owner at a
given event time for a flow action event.
+ * After acquiring the lease, the host can pursue executing the action. Once
it has completed this action, it
+ * marks the lease as completed by calling the
+ * {@link
MultiActiveLeaseArbiter#recordLeaseSuccess(MultiActiveLeaseArbiter.LeaseObtainedStatus)}
method. Hosts
+ * that fail to acquire a lease will use the {@link
DagActionReminderScheduler} to set a reminder for the flow action
+ * event to check back in on the previous lease owner's completion status.
+ */
+@Slf4j
+public class ReminderSettingDagProcLeaseArbiter implements
MultiActiveLeaseArbiter {
+ private final MultiActiveLeaseArbiter decoratedLeaseArbiter;
+ private final DagActionReminderScheduler _dagActionReminderScheduler;
Review Comment:
drop leading `_`
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/InstrumentedLeaseArbiter.java:
##########
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.io.IOException;
+
+import com.typesafe.config.Config;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.gobblin.instrumented.Instrumented;
+import org.apache.gobblin.metrics.ContextAwareCounter;
+import org.apache.gobblin.metrics.ContextAwareMeter;
+import org.apache.gobblin.metrics.MetricContext;
+import org.apache.gobblin.metrics.ServiceMetricNames;
+import org.apache.gobblin.runtime.api.DagActionStore;
+import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter;
+import org.apache.gobblin.util.ConfigUtils;
+
+
+/*
+ A generic lease arbitration decorator built upon the {@link
MultiActiveLeaseArbiter} which encapsulates common
+ functionality desired by lease arbiter users to track metrics on all lease
attempts made by the arbiter. The metrics
+ can be used to compare relative performance among arbitration participants.
+ */
+@Slf4j
+public class InstrumentedLeaseArbiter implements MultiActiveLeaseArbiter {
+ protected MultiActiveLeaseArbiter decoratedMultiActiveLeaseArbiter;
+ @Getter
+ protected MetricContext metricContext;
+ private ContextAwareCounter leaseObtainedCount;
+
+ private ContextAwareCounter leasedToAnotherStatusCount;
+
+ private ContextAwareCounter noLongerLeasingStatusCount;
+ private ContextAwareMeter leasesObtainedDueToReminderCount;
+
+ public InstrumentedLeaseArbiter(Config config, MultiActiveLeaseArbiter
leaseDeterminationStore,
+ String metricsPrefix) {
+ this.decoratedMultiActiveLeaseArbiter = leaseDeterminationStore;
+ this.metricContext = Instrumented.getMetricContext(new
org.apache.gobblin.configuration.State(ConfigUtils.configToProperties(config)),
+ this.getClass());
+ initializeMetrics(metricsPrefix);
+ }
+
+ private void initializeMetrics(String metricsPrefix) {
+ // If a valid metrics prefix is provided then add a delimiter after it
+ if (!metricsPrefix.equals("")) {
+ metricsPrefix += ".";
+ }
+ this.leaseObtainedCount =
this.metricContext.contextAwareCounter(metricsPrefix +
ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASE_OBTAINED_COUNT);
+ this.leasedToAnotherStatusCount =
this.metricContext.contextAwareCounter(metricsPrefix +
ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASED_TO_ANOTHER_COUNT);
+ this.noLongerLeasingStatusCount =
this.metricContext.contextAwareCounter(metricsPrefix +
ServiceMetricNames.FLOW_TRIGGER_HANDLER_NO_LONGER_LEASING_COUNT);
+ this.leasesObtainedDueToReminderCount =
this.metricContext.contextAwareMeter(metricsPrefix +
ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASES_OBTAINED_DUE_TO_REMINDER_COUNT);
+ }
+
+ @Override
+ public MultiActiveLeaseArbiter.LeaseAttemptStatus
tryAcquireLease(DagActionStore.DagAction flowAction, long eventTimeMillis,
+ boolean isReminderEvent, boolean skipFlowExecutionIdReplacement) throws
IOException {
+
+ MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus =
+ decoratedMultiActiveLeaseArbiter.tryAcquireLease(flowAction,
eventTimeMillis, isReminderEvent,
+ skipFlowExecutionIdReplacement);
+ log.info("Multi-active scheduler lease attempt for dagAction: {} received
type of leaseAttemptStatus: [{}, "
+ + "eventTimestamp: {}] ", flowAction,
leaseAttemptStatus.getClass().getName(), eventTimeMillis);
+ if (leaseAttemptStatus instanceof
MultiActiveLeaseArbiter.LeaseObtainedStatus) {
+ if (isReminderEvent) {
+ this.leasesObtainedDueToReminderCount.mark();
+ }
+ this.leaseObtainedCount.inc();
+ return leaseAttemptStatus;
+ } else if (leaseAttemptStatus instanceof
MultiActiveLeaseArbiter.LeasedToAnotherStatus) {
+ this.leasedToAnotherStatusCount.inc();
+ return leaseAttemptStatus;
+ } else if (leaseAttemptStatus instanceof
MultiActiveLeaseArbiter.NoLongerLeasingStatus) {
+ this.noLongerLeasingStatusCount.inc();
+ return leaseAttemptStatus;
+ }
+ throw new RuntimeException(String.format("Received type of
leaseAttemptStatus: %s not handled by this method",
+ leaseAttemptStatus.getClass().getName()));
+ }
+
+ @Override
+ public boolean recordLeaseSuccess(LeaseObtainedStatus status)
+ throws IOException {
+ return this.decoratedMultiActiveLeaseArbiter.recordLeaseSuccess(status);
Review Comment:
measure how many times true vs false?
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/Orchestrator.java:
##########
@@ -230,15 +230,16 @@ public void orchestrate(Spec spec, Properties jobProps,
long triggerTimestampMil
Map<String, String> flowMetadata =
TimingEventUtils.getFlowMetadata(flowSpec);
String flowExecutionId =
String.valueOf(FlowUtils.getOrCreateFlowExecutionId(flowSpec));
+ // Note we use "" as jobName for lease arbitration since orchestration
is at a flow level
Review Comment:
suggest named constant for use of `""` for this particular (`jobName`)
purpose
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ReminderSettingFlowTriggerLeaseArbiter.java:
##########
@@ -332,4 +303,17 @@ protected static long getUTCTimeFromDelayPeriod(long
delayPeriodMillis) {
Date date = Date.from(localDateTime.atZone(ZoneId.of("UTC")).toInstant());
return GobblinServiceJobScheduler.utcDateAsUTCEpochMillis(date);
}
+
+ @Override
+ public LeaseAttemptStatus tryAcquireLease(DagActionStore.DagAction
dagAction, long eventTimeMillis,
+ boolean isReminderEvent, boolean adoptConsensusFlowExecutionId)
+ throws IOException {
+ return this.decoratedLeaseArbiter.tryAcquireLease(dagAction,
eventTimeMillis, isReminderEvent, adoptConsensusFlowExecutionId);
+ }
+
+ @Override
+ public boolean recordLeaseSuccess(LeaseObtainedStatus status)
+ throws IOException {
+ return this.decoratedLeaseArbiter.recordLeaseSuccess(status);
+ }
Review Comment:
this abstraction seems ill-suited to the Decorator pattern, as there's no
decoration going on
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/InstrumentedLeaseArbiter.java:
##########
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.io.IOException;
+
+import com.typesafe.config.Config;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.gobblin.instrumented.Instrumented;
+import org.apache.gobblin.metrics.ContextAwareCounter;
+import org.apache.gobblin.metrics.ContextAwareMeter;
+import org.apache.gobblin.metrics.MetricContext;
+import org.apache.gobblin.metrics.ServiceMetricNames;
+import org.apache.gobblin.runtime.api.DagActionStore;
+import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter;
+import org.apache.gobblin.util.ConfigUtils;
+
+
+/*
+ A generic lease arbitration decorator built upon the {@link
MultiActiveLeaseArbiter} which encapsulates common
+ functionality desired by lease arbiter users to track metrics on all lease
attempts made by the arbiter. The metrics
+ can be used to compare relative performance among arbitration participants.
+ */
+@Slf4j
+public class InstrumentedLeaseArbiter implements MultiActiveLeaseArbiter {
+ protected MultiActiveLeaseArbiter decoratedMultiActiveLeaseArbiter;
+ @Getter
+ protected MetricContext metricContext;
+ private ContextAwareCounter leaseObtainedCount;
+
+ private ContextAwareCounter leasedToAnotherStatusCount;
+
+ private ContextAwareCounter noLongerLeasingStatusCount;
+ private ContextAwareMeter leasesObtainedDueToReminderCount;
+
+ public InstrumentedLeaseArbiter(Config config, MultiActiveLeaseArbiter
leaseDeterminationStore,
+ String metricsPrefix) {
+ this.decoratedMultiActiveLeaseArbiter = leaseDeterminationStore;
+ this.metricContext = Instrumented.getMetricContext(new
org.apache.gobblin.configuration.State(ConfigUtils.configToProperties(config)),
+ this.getClass());
+ initializeMetrics(metricsPrefix);
+ }
+
+ private void initializeMetrics(String metricsPrefix) {
+ // If a valid metrics prefix is provided then add a delimiter after it
+ if (!metricsPrefix.equals("")) {
+ metricsPrefix += ".";
+ }
+ this.leaseObtainedCount =
this.metricContext.contextAwareCounter(metricsPrefix +
ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASE_OBTAINED_COUNT);
+ this.leasedToAnotherStatusCount =
this.metricContext.contextAwareCounter(metricsPrefix +
ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASED_TO_ANOTHER_COUNT);
+ this.noLongerLeasingStatusCount =
this.metricContext.contextAwareCounter(metricsPrefix +
ServiceMetricNames.FLOW_TRIGGER_HANDLER_NO_LONGER_LEASING_COUNT);
+ this.leasesObtainedDueToReminderCount =
this.metricContext.contextAwareMeter(metricsPrefix +
ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASES_OBTAINED_DUE_TO_REMINDER_COUNT);
+ }
+
+ @Override
+ public MultiActiveLeaseArbiter.LeaseAttemptStatus
tryAcquireLease(DagActionStore.DagAction flowAction, long eventTimeMillis,
Review Comment:
nit: rename `flowAction` => `dagAction`
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/InstrumentedLeaseArbiter.java:
##########
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.io.IOException;
+
+import com.typesafe.config.Config;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.gobblin.instrumented.Instrumented;
+import org.apache.gobblin.metrics.ContextAwareCounter;
+import org.apache.gobblin.metrics.ContextAwareMeter;
+import org.apache.gobblin.metrics.MetricContext;
+import org.apache.gobblin.metrics.ServiceMetricNames;
+import org.apache.gobblin.runtime.api.DagActionStore;
+import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter;
+import org.apache.gobblin.util.ConfigUtils;
+
+
+/*
+ A generic lease arbitration decorator built upon the {@link
MultiActiveLeaseArbiter} which encapsulates common
+ functionality desired by lease arbiter users to track metrics on all lease
attempts made by the arbiter. The metrics
Review Comment:
I'd whittle:
> A generic lease arbitration decorator built upon the {@link
MultiActiveLeaseArbiter} which encapsulates common functionality desired by
lease arbiter users to track metrics on all lease attempts made by the arbiter.
down to:
> A {@link MultiActiveLeaseArbiter} decorator for tracking metrics about
lease attempts.
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/InstrumentedLeaseArbiter.java:
##########
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.io.IOException;
+
+import com.typesafe.config.Config;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.gobblin.instrumented.Instrumented;
+import org.apache.gobblin.metrics.ContextAwareCounter;
+import org.apache.gobblin.metrics.ContextAwareMeter;
+import org.apache.gobblin.metrics.MetricContext;
+import org.apache.gobblin.metrics.ServiceMetricNames;
+import org.apache.gobblin.runtime.api.DagActionStore;
+import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter;
+import org.apache.gobblin.util.ConfigUtils;
+
+
+/*
+ A generic lease arbitration decorator built upon the {@link
MultiActiveLeaseArbiter} which encapsulates common
+ functionality desired by lease arbiter users to track metrics on all lease
attempts made by the arbiter. The metrics
+ can be used to compare relative performance among arbitration participants.
+ */
+@Slf4j
+public class InstrumentedLeaseArbiter implements MultiActiveLeaseArbiter {
+ protected MultiActiveLeaseArbiter decoratedMultiActiveLeaseArbiter;
+ @Getter
+ protected MetricContext metricContext;
+ private ContextAwareCounter leaseObtainedCount;
+
+ private ContextAwareCounter leasedToAnotherStatusCount;
+
+ private ContextAwareCounter noLongerLeasingStatusCount;
+ private ContextAwareMeter leasesObtainedDueToReminderCount;
+
+ public InstrumentedLeaseArbiter(Config config, MultiActiveLeaseArbiter
leaseDeterminationStore,
+ String metricsPrefix) {
+ this.decoratedMultiActiveLeaseArbiter = leaseDeterminationStore;
+ this.metricContext = Instrumented.getMetricContext(new
org.apache.gobblin.configuration.State(ConfigUtils.configToProperties(config)),
+ this.getClass());
+ initializeMetrics(metricsPrefix);
+ }
+
+ private void initializeMetrics(String metricsPrefix) {
+ // If a valid metrics prefix is provided then add a delimiter after it
+ if (!metricsPrefix.equals("")) {
+ metricsPrefix += ".";
+ }
Review Comment:
I slightly prefer `.isEmpty()` to `.equals("")`
also `.concat(".")`
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ReminderSettingDagProcLeaseArbiter.java:
##########
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.quartz.Job;
+import org.quartz.JobBuilder;
+import org.quartz.JobDataMap;
+import org.quartz.JobDetail;
+import org.quartz.JobExecutionContext;
+import org.quartz.SchedulerException;
+import org.quartz.Trigger;
+import org.quartz.TriggerBuilder;
+
+import com.typesafe.config.Config;
+
+import javax.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.gobblin.configuration.ConfigurationKeys;
+import org.apache.gobblin.runtime.api.DagActionStore;
+import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter;
+
+
+/**
+ * Decorator used to coordinate multiple hosts with execution components
enabled to respond to flow action events with
+ * added capabilities to properly handle the result of attempted ownership
over these flow action events. It uses the
+ * {@link MultiActiveLeaseArbiter} to determine a single lease owner at a
given event time for a flow action event.
+ * After acquiring the lease, the host can pursue executing the action. Once
it has completed this action, it
+ * marks the lease as completed by calling the
+ * {@link
MultiActiveLeaseArbiter#recordLeaseSuccess(MultiActiveLeaseArbiter.LeaseObtainedStatus)}
method. Hosts
+ * that fail to acquire a lease will use the {@link
DagActionReminderScheduler} to set a reminder for the flow action
+ * event to check back in on the previous lease owner's completion status.
+ */
+@Slf4j
+public class ReminderSettingDagProcLeaseArbiter implements
MultiActiveLeaseArbiter {
+ private final MultiActiveLeaseArbiter decoratedLeaseArbiter;
+ private final DagActionReminderScheduler _dagActionReminderScheduler;
+ private final Config config;
+
+ @Inject
+ public ReminderSettingDagProcLeaseArbiter(Config config,
MultiActiveLeaseArbiter leaseArbiter, DagActionReminderScheduler
dagActionReminderScheduler) throws IOException {
+ this.decoratedLeaseArbiter = leaseArbiter;
+ this._dagActionReminderScheduler = dagActionReminderScheduler;
+ this.config = config;
+ }
+
+ /**
+ * This method is used by the multi-active scheduler and multi-active
execution classes (DagTaskStream) to attempt a
+ * lease for a particular job event and return the status of the attempt.
+ * @param flowAction
+ * @param eventTimeMillis
+ * @param isReminderEvent
+ * @param skipFlowExecutionIdReplacement
+ * @return
+ */
+ @Override
+ public MultiActiveLeaseArbiter.LeaseAttemptStatus
tryAcquireLease(DagActionStore.DagAction flowAction, long eventTimeMillis,
+ boolean isReminderEvent, boolean skipFlowExecutionIdReplacement) {
+ try {
+ MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus =
this.tryAcquireLease(flowAction, eventTimeMillis, isReminderEvent,
skipFlowExecutionIdReplacement);
+ /* Schedule a reminder for the event unless the lease has been completed
to safeguard against case lease owner
+ fails to complete lease
+ */
+ if (leaseAttemptStatus instanceof
MultiActiveLeaseArbiter.LeaseObtainedStatus) {
+ scheduleReminderForEvent((LeaseObtainedStatus) leaseAttemptStatus);
+ } else if (leaseAttemptStatus instanceof
MultiActiveLeaseArbiter.LeasedToAnotherStatus) {
+ scheduleReminderForEvent((LeasedToAnotherStatus) leaseAttemptStatus);
+ }
+ return leaseAttemptStatus;
+ } catch (SchedulerException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean recordLeaseSuccess(LeaseObtainedStatus status)
+ throws IOException {
+ return this.decoratedLeaseArbiter.recordLeaseSuccess(status);
+ }
+
+ protected void
scheduleReminderForEvent(MultiActiveLeaseArbiter.LeasedToAnotherStatus
leaseStatus)
+ throws SchedulerException {
+ _dagActionReminderScheduler.scheduleReminder(leaseStatus.getDagAction(),
leaseStatus.getMinimumLingerDurationMillis());
+ }
+
+ protected void
scheduleReminderForEvent(MultiActiveLeaseArbiter.LeaseObtainedStatus
leaseStatus)
+ throws SchedulerException {
+ _dagActionReminderScheduler.scheduleReminder(leaseStatus.getDagAction(),
leaseStatus.getMinimumLingerDurationMillis());
+ }
Review Comment:
I don't like the duplication... and I agree there's no perfect solution here
(in the java type system).
but casting's imperfect on its own, so I'd probably bite the bullet and add
*concrete impls* to the `LeaseAttemptStatus` that both return `null`:
```
public DagAction getDagAction() ...
public long getMinimumLingerDurationMillis() ...
```
thereafter the `@Data` or `@Getter`s on these two derived classes will
override to a real value for each.
thereafter, the conditions above become merely:
```
if (!leaseAttemptStatus instanceof MultiActiveLeaseArbiter.NoLongerLeasing) {
scheduleReminderForEvent(leaseAttemptStatus);
}
...
```
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ReminderSettingDagProcLeaseArbiter.java:
##########
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.quartz.Job;
+import org.quartz.JobBuilder;
+import org.quartz.JobDataMap;
+import org.quartz.JobDetail;
+import org.quartz.JobExecutionContext;
+import org.quartz.SchedulerException;
+import org.quartz.Trigger;
+import org.quartz.TriggerBuilder;
+
+import com.typesafe.config.Config;
+
+import javax.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.gobblin.configuration.ConfigurationKeys;
+import org.apache.gobblin.runtime.api.DagActionStore;
+import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter;
+
+
+/**
+ * Decorator used to coordinate multiple hosts with execution components
enabled to respond to flow action events with
+ * added capabilities to properly handle the result of attempted ownership
over these flow action events. It uses the
+ * {@link MultiActiveLeaseArbiter} to determine a single lease owner at a
given event time for a flow action event.
+ * After acquiring the lease, the host can pursue executing the action. Once
it has completed this action, it
+ * marks the lease as completed by calling the
+ * {@link
MultiActiveLeaseArbiter#recordLeaseSuccess(MultiActiveLeaseArbiter.LeaseObtainedStatus)}
method. Hosts
+ * that fail to acquire a lease will use the {@link
DagActionReminderScheduler} to set a reminder for the flow action
Review Comment:
suggest to cut out most of this, since it's already known from the base
class/interface javadoc. no benefit to repeat here!
the critical new info to communicate is this last sentence about the setting
of reminders. be sure to clarify the contract--a reminder *unless* `{@link
NoLongerLeasing}`
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/InstrumentedLeaseArbiter.java:
##########
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.io.IOException;
+
+import com.typesafe.config.Config;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.gobblin.instrumented.Instrumented;
+import org.apache.gobblin.metrics.ContextAwareCounter;
+import org.apache.gobblin.metrics.ContextAwareMeter;
+import org.apache.gobblin.metrics.MetricContext;
+import org.apache.gobblin.metrics.ServiceMetricNames;
+import org.apache.gobblin.runtime.api.DagActionStore;
+import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter;
+import org.apache.gobblin.util.ConfigUtils;
+
+
+/*
+ A generic lease arbitration decorator built upon the {@link
MultiActiveLeaseArbiter} which encapsulates common
+ functionality desired by lease arbiter users to track metrics on all lease
attempts made by the arbiter. The metrics
+ can be used to compare relative performance among arbitration participants.
+ */
+@Slf4j
+public class InstrumentedLeaseArbiter implements MultiActiveLeaseArbiter {
+ protected MultiActiveLeaseArbiter decoratedMultiActiveLeaseArbiter;
+ @Getter
+ protected MetricContext metricContext;
+ private ContextAwareCounter leaseObtainedCount;
+
+ private ContextAwareCounter leasedToAnotherStatusCount;
+
+ private ContextAwareCounter noLongerLeasingStatusCount;
+ private ContextAwareMeter leasesObtainedDueToReminderCount;
+
+ public InstrumentedLeaseArbiter(Config config, MultiActiveLeaseArbiter
leaseDeterminationStore,
+ String metricsPrefix) {
+ this.decoratedMultiActiveLeaseArbiter = leaseDeterminationStore;
+ this.metricContext = Instrumented.getMetricContext(new
org.apache.gobblin.configuration.State(ConfigUtils.configToProperties(config)),
+ this.getClass());
+ initializeMetrics(metricsPrefix);
+ }
+
+ private void initializeMetrics(String metricsPrefix) {
+ // If a valid metrics prefix is provided then add a delimiter after it
+ if (!metricsPrefix.equals("")) {
+ metricsPrefix += ".";
+ }
+ this.leaseObtainedCount =
this.metricContext.contextAwareCounter(metricsPrefix +
ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASE_OBTAINED_COUNT);
+ this.leasedToAnotherStatusCount =
this.metricContext.contextAwareCounter(metricsPrefix +
ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASED_TO_ANOTHER_COUNT);
+ this.noLongerLeasingStatusCount =
this.metricContext.contextAwareCounter(metricsPrefix +
ServiceMetricNames.FLOW_TRIGGER_HANDLER_NO_LONGER_LEASING_COUNT);
+ this.leasesObtainedDueToReminderCount =
this.metricContext.contextAwareMeter(metricsPrefix +
ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASES_OBTAINED_DUE_TO_REMINDER_COUNT);
+ }
+
+ @Override
+ public MultiActiveLeaseArbiter.LeaseAttemptStatus
tryAcquireLease(DagActionStore.DagAction flowAction, long eventTimeMillis,
+ boolean isReminderEvent, boolean skipFlowExecutionIdReplacement) throws
IOException {
+
+ MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus =
+ decoratedMultiActiveLeaseArbiter.tryAcquireLease(flowAction,
eventTimeMillis, isReminderEvent,
+ skipFlowExecutionIdReplacement);
+ log.info("Multi-active scheduler lease attempt for dagAction: {} received
type of leaseAttemptStatus: [{}, "
+ + "eventTimestamp: {}] ", flowAction,
leaseAttemptStatus.getClass().getName(), eventTimeMillis);
Review Comment:
such logging rarely belongs in a decorator. typically the underlying
"decorated" class would do this, which would make this duplicative
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ReminderSettingDagProcLeaseArbiter.java:
##########
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.quartz.Job;
+import org.quartz.JobBuilder;
+import org.quartz.JobDataMap;
+import org.quartz.JobDetail;
+import org.quartz.JobExecutionContext;
+import org.quartz.SchedulerException;
+import org.quartz.Trigger;
+import org.quartz.TriggerBuilder;
+
+import com.typesafe.config.Config;
+
+import javax.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.gobblin.configuration.ConfigurationKeys;
+import org.apache.gobblin.runtime.api.DagActionStore;
+import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter;
+
+
+/**
+ * Decorator used to coordinate multiple hosts with execution components
enabled to respond to flow action events with
+ * added capabilities to properly handle the result of attempted ownership
over these flow action events. It uses the
+ * {@link MultiActiveLeaseArbiter} to determine a single lease owner at a
given event time for a flow action event.
+ * After acquiring the lease, the host can pursue executing the action. Once
it has completed this action, it
+ * marks the lease as completed by calling the
+ * {@link
MultiActiveLeaseArbiter#recordLeaseSuccess(MultiActiveLeaseArbiter.LeaseObtainedStatus)}
method. Hosts
+ * that fail to acquire a lease will use the {@link
DagActionReminderScheduler} to set a reminder for the flow action
+ * event to check back in on the previous lease owner's completion status.
+ */
+@Slf4j
+public class ReminderSettingDagProcLeaseArbiter implements
MultiActiveLeaseArbiter {
+ private final MultiActiveLeaseArbiter decoratedLeaseArbiter;
+ private final DagActionReminderScheduler _dagActionReminderScheduler;
+ private final Config config;
+
+ @Inject
+ public ReminderSettingDagProcLeaseArbiter(Config config,
MultiActiveLeaseArbiter leaseArbiter, DagActionReminderScheduler
dagActionReminderScheduler) throws IOException {
+ this.decoratedLeaseArbiter = leaseArbiter;
+ this._dagActionReminderScheduler = dagActionReminderScheduler;
+ this.config = config;
+ }
+
+ /**
+ * This method is used by the multi-active scheduler and multi-active
execution classes (DagTaskStream) to attempt a
+ * lease for a particular job event and return the status of the attempt.
+ * @param flowAction
+ * @param eventTimeMillis
+ * @param isReminderEvent
+ * @param skipFlowExecutionIdReplacement
+ * @return
+ */
+ @Override
+ public MultiActiveLeaseArbiter.LeaseAttemptStatus
tryAcquireLease(DagActionStore.DagAction flowAction, long eventTimeMillis,
+ boolean isReminderEvent, boolean skipFlowExecutionIdReplacement) {
+ try {
+ MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus =
this.tryAcquireLease(flowAction, eventTimeMillis, isReminderEvent,
skipFlowExecutionIdReplacement);
+ /* Schedule a reminder for the event unless the lease has been completed
to safeguard against case lease owner
Review Comment:
to be more specific, I believe you mean:
"to safeguard against the case where even we, as the now-current lease owner
fail to complete processing"
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ReminderSettingFlowTriggerLeaseArbiter.java:
##########
@@ -64,133 +65,103 @@
* in failure cases.
*/
@Slf4j
-public class FlowTriggerHandler {
+public class ReminderSettingFlowTriggerLeaseArbiter implements
MultiActiveLeaseArbiter {
+ private final MultiActiveLeaseArbiter decoratedLeaseArbiter;
+ private Optional<DagActionStore> dagActionStore;
+ private final MetricContext metricContext;
private final int schedulerMaxBackoffMillis;
private static Random random = new Random();
- protected Optional<MultiActiveLeaseArbiter> multiActiveLeaseArbiter;
protected SchedulerService schedulerService;
- protected Optional<DagActionStore> dagActionStore;
- private MetricContext metricContext;
private ContextAwareMeter numFlowsSubmitted;
-
- private ContextAwareCounter leaseObtainedCount;
-
- private ContextAwareCounter leasedToAnotherStatusCount;
-
- private ContextAwareCounter noLongerLeasingStatusCount;
private ContextAwareCounter jobDoesNotExistInSchedulerCount;
private ContextAwareCounter failedToSetEventReminderCount;
- private ContextAwareMeter leasesObtainedDueToReminderCount;
private ContextAwareMeter failedToRecordLeaseSuccessCount;
private ContextAwareMeter recordedLeaseSuccessCount;
@Inject
- public FlowTriggerHandler(Config config, Optional<MultiActiveLeaseArbiter>
leaseDeterminationStore,
- SchedulerService schedulerService, Optional<DagActionStore>
dagActionStore) {
+ public ReminderSettingFlowTriggerLeaseArbiter(Config config,
MultiActiveLeaseArbiter leaseArbiter,
+ SchedulerService schedulerService, Optional<DagActionStore>
dagActionStore)
+ throws IOException {
+ this.decoratedLeaseArbiter = leaseArbiter;
+ this.dagActionStore = dagActionStore;
this.schedulerMaxBackoffMillis = ConfigUtils.getInt(config,
ConfigurationKeys.SCHEDULER_MAX_BACKOFF_MILLIS_KEY,
ConfigurationKeys.DEFAULT_SCHEDULER_MAX_BACKOFF_MILLIS);
- this.multiActiveLeaseArbiter = leaseDeterminationStore;
this.schedulerService = schedulerService;
- this.dagActionStore = dagActionStore;
+
+ // Initialize FlowTriggerDecorator related metrics
Review Comment:
old name?
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ReminderSettingDagProcLeaseArbiter.java:
##########
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.quartz.Job;
+import org.quartz.JobBuilder;
+import org.quartz.JobDataMap;
+import org.quartz.JobDetail;
+import org.quartz.JobExecutionContext;
+import org.quartz.SchedulerException;
+import org.quartz.Trigger;
+import org.quartz.TriggerBuilder;
+
+import com.typesafe.config.Config;
+
+import javax.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.gobblin.configuration.ConfigurationKeys;
+import org.apache.gobblin.runtime.api.DagActionStore;
+import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter;
+
+
+/**
+ * Decorator used to coordinate multiple hosts with execution components
enabled to respond to flow action events with
+ * added capabilities to properly handle the result of attempted ownership
over these flow action events. It uses the
+ * {@link MultiActiveLeaseArbiter} to determine a single lease owner at a
given event time for a flow action event.
+ * After acquiring the lease, the host can pursue executing the action. Once
it has completed this action, it
+ * marks the lease as completed by calling the
+ * {@link
MultiActiveLeaseArbiter#recordLeaseSuccess(MultiActiveLeaseArbiter.LeaseObtainedStatus)}
method. Hosts
+ * that fail to acquire a lease will use the {@link
DagActionReminderScheduler} to set a reminder for the flow action
+ * event to check back in on the previous lease owner's completion status.
+ */
+@Slf4j
+public class ReminderSettingDagProcLeaseArbiter implements
MultiActiveLeaseArbiter {
+ private final MultiActiveLeaseArbiter decoratedLeaseArbiter;
+ private final DagActionReminderScheduler _dagActionReminderScheduler;
+ private final Config config;
+
+ @Inject
+ public ReminderSettingDagProcLeaseArbiter(Config config,
MultiActiveLeaseArbiter leaseArbiter, DagActionReminderScheduler
dagActionReminderScheduler) throws IOException {
+ this.decoratedLeaseArbiter = leaseArbiter;
+ this._dagActionReminderScheduler = dagActionReminderScheduler;
+ this.config = config;
+ }
+
+ /**
+ * This method is used by the multi-active scheduler and multi-active
execution classes (DagTaskStream) to attempt a
+ * lease for a particular job event and return the status of the attempt.
+ * @param flowAction
+ * @param eventTimeMillis
+ * @param isReminderEvent
+ * @param skipFlowExecutionIdReplacement
+ * @return
+ */
+ @Override
+ public MultiActiveLeaseArbiter.LeaseAttemptStatus
tryAcquireLease(DagActionStore.DagAction flowAction, long eventTimeMillis,
+ boolean isReminderEvent, boolean skipFlowExecutionIdReplacement) {
+ try {
+ MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus =
this.tryAcquireLease(flowAction, eventTimeMillis, isReminderEvent,
skipFlowExecutionIdReplacement);
+ /* Schedule a reminder for the event unless the lease has been completed
to safeguard against case lease owner
+ fails to complete lease
+ */
+ if (leaseAttemptStatus instanceof
MultiActiveLeaseArbiter.LeaseObtainedStatus) {
+ scheduleReminderForEvent((LeaseObtainedStatus) leaseAttemptStatus);
+ } else if (leaseAttemptStatus instanceof
MultiActiveLeaseArbiter.LeasedToAnotherStatus) {
+ scheduleReminderForEvent((LeasedToAnotherStatus) leaseAttemptStatus);
+ }
+ return leaseAttemptStatus;
+ } catch (SchedulerException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean recordLeaseSuccess(LeaseObtainedStatus status)
+ throws IOException {
+ return this.decoratedLeaseArbiter.recordLeaseSuccess(status);
+ }
+
+ protected void
scheduleReminderForEvent(MultiActiveLeaseArbiter.LeasedToAnotherStatus
leaseStatus)
+ throws SchedulerException {
+ _dagActionReminderScheduler.scheduleReminder(leaseStatus.getDagAction(),
leaseStatus.getMinimumLingerDurationMillis());
+ }
+
+ protected void
scheduleReminderForEvent(MultiActiveLeaseArbiter.LeaseObtainedStatus
leaseStatus)
+ throws SchedulerException {
+ _dagActionReminderScheduler.scheduleReminder(leaseStatus.getDagAction(),
leaseStatus.getMinimumLingerDurationMillis());
+ }
+
+ @Slf4j
+ public static class ReminderJob implements Job {
Review Comment:
let's add javadoc to this and each of the `public static` methods that follow
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ReminderSettingDagProcLeaseArbiter.java:
##########
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.quartz.Job;
+import org.quartz.JobBuilder;
+import org.quartz.JobDataMap;
+import org.quartz.JobDetail;
+import org.quartz.JobExecutionContext;
+import org.quartz.SchedulerException;
+import org.quartz.Trigger;
+import org.quartz.TriggerBuilder;
+
+import com.typesafe.config.Config;
+
+import javax.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.gobblin.configuration.ConfigurationKeys;
+import org.apache.gobblin.runtime.api.DagActionStore;
+import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter;
+
+
+/**
+ * Decorator used to coordinate multiple hosts with execution components
enabled to respond to flow action events with
+ * added capabilities to properly handle the result of attempted ownership
over these flow action events. It uses the
+ * {@link MultiActiveLeaseArbiter} to determine a single lease owner at a
given event time for a flow action event.
+ * After acquiring the lease, the host can pursue executing the action. Once
it has completed this action, it
+ * marks the lease as completed by calling the
+ * {@link
MultiActiveLeaseArbiter#recordLeaseSuccess(MultiActiveLeaseArbiter.LeaseObtainedStatus)}
method. Hosts
+ * that fail to acquire a lease will use the {@link
DagActionReminderScheduler} to set a reminder for the flow action
+ * event to check back in on the previous lease owner's completion status.
+ */
+@Slf4j
+public class ReminderSettingDagProcLeaseArbiter implements
MultiActiveLeaseArbiter {
+ private final MultiActiveLeaseArbiter decoratedLeaseArbiter;
+ private final DagActionReminderScheduler _dagActionReminderScheduler;
+ private final Config config;
+
+ @Inject
+ public ReminderSettingDagProcLeaseArbiter(Config config,
MultiActiveLeaseArbiter leaseArbiter, DagActionReminderScheduler
dagActionReminderScheduler) throws IOException {
+ this.decoratedLeaseArbiter = leaseArbiter;
+ this._dagActionReminderScheduler = dagActionReminderScheduler;
+ this.config = config;
+ }
+
+ /**
+ * This method is used by the multi-active scheduler and multi-active
execution classes (DagTaskStream) to attempt a
+ * lease for a particular job event and return the status of the attempt.
+ * @param flowAction
+ * @param eventTimeMillis
+ * @param isReminderEvent
+ * @param skipFlowExecutionIdReplacement
+ * @return
+ */
+ @Override
+ public MultiActiveLeaseArbiter.LeaseAttemptStatus
tryAcquireLease(DagActionStore.DagAction flowAction, long eventTimeMillis,
+ boolean isReminderEvent, boolean skipFlowExecutionIdReplacement) {
+ try {
+ MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus =
this.tryAcquireLease(flowAction, eventTimeMillis, isReminderEvent,
skipFlowExecutionIdReplacement);
+ /* Schedule a reminder for the event unless the lease has been completed
to safeguard against case lease owner
+ fails to complete lease
+ */
+ if (leaseAttemptStatus instanceof
MultiActiveLeaseArbiter.LeaseObtainedStatus) {
+ scheduleReminderForEvent((LeaseObtainedStatus) leaseAttemptStatus);
+ } else if (leaseAttemptStatus instanceof
MultiActiveLeaseArbiter.LeasedToAnotherStatus) {
+ scheduleReminderForEvent((LeasedToAnotherStatus) leaseAttemptStatus);
+ }
+ return leaseAttemptStatus;
+ } catch (SchedulerException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean recordLeaseSuccess(LeaseObtainedStatus status)
+ throws IOException {
+ return this.decoratedLeaseArbiter.recordLeaseSuccess(status);
+ }
+
+ protected void
scheduleReminderForEvent(MultiActiveLeaseArbiter.LeasedToAnotherStatus
leaseStatus)
+ throws SchedulerException {
+ _dagActionReminderScheduler.scheduleReminder(leaseStatus.getDagAction(),
leaseStatus.getMinimumLingerDurationMillis());
+ }
+
+ protected void
scheduleReminderForEvent(MultiActiveLeaseArbiter.LeaseObtainedStatus
leaseStatus)
+ throws SchedulerException {
+ _dagActionReminderScheduler.scheduleReminder(leaseStatus.getDagAction(),
leaseStatus.getMinimumLingerDurationMillis());
+ }
+
+ @Slf4j
+ public static class ReminderJob implements Job {
+ public static final String FLOW_ACTION_TYPE_KEY = "flow.actionType";
+ public static final String DAG_MANAGEMENT_KEY = "dag.management";
+
+ @Override
+ public void execute(JobExecutionContext context) {
+ // Get properties from the trigger to create a dagAction
+ JobDataMap jobDataMap = context.getTrigger().getJobDataMap();
+ String flowName = jobDataMap.getString(ConfigurationKeys.FLOW_NAME_KEY);
+ String flowGroup =
jobDataMap.getString(ConfigurationKeys.FLOW_GROUP_KEY);
+ String jobName = jobDataMap.getString(ConfigurationKeys.JOB_NAME_KEY);
+ String flowId =
jobDataMap.getString(ConfigurationKeys.FLOW_EXECUTION_ID_KEY);
+ DagActionStore.DagActionType dagActionType =
DagActionStore.DagActionType.valueOf(
+ jobDataMap.getString(FLOW_ACTION_TYPE_KEY));
+ DagManagement dagManagement = (DagManagement)
jobDataMap.get(DAG_MANAGEMENT_KEY);
Review Comment:
I really would like to avoid this (de)serialization. perhaps we should
explore a `@Singleton` with an accessor for the "global" `DagManagement`
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ReminderSettingDagProcLeaseArbiter.java:
##########
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.quartz.Job;
+import org.quartz.JobBuilder;
+import org.quartz.JobDataMap;
+import org.quartz.JobDetail;
+import org.quartz.JobExecutionContext;
+import org.quartz.SchedulerException;
+import org.quartz.Trigger;
+import org.quartz.TriggerBuilder;
+
+import com.typesafe.config.Config;
+
+import javax.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.gobblin.configuration.ConfigurationKeys;
+import org.apache.gobblin.runtime.api.DagActionStore;
+import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter;
+
+
+/**
+ * Decorator used to coordinate multiple hosts with execution components
enabled to respond to flow action events with
+ * added capabilities to properly handle the result of attempted ownership
over these flow action events. It uses the
+ * {@link MultiActiveLeaseArbiter} to determine a single lease owner at a
given event time for a flow action event.
+ * After acquiring the lease, the host can pursue executing the action. Once
it has completed this action, it
+ * marks the lease as completed by calling the
+ * {@link
MultiActiveLeaseArbiter#recordLeaseSuccess(MultiActiveLeaseArbiter.LeaseObtainedStatus)}
method. Hosts
+ * that fail to acquire a lease will use the {@link
DagActionReminderScheduler} to set a reminder for the flow action
+ * event to check back in on the previous lease owner's completion status.
Review Comment:
perhaps more accurate to say: "to reattempt the lease after the current
lease holder's grant would have expired"
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/ReminderSettingDagProcLeaseArbiter.java:
##########
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.orchestration;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.quartz.Job;
+import org.quartz.JobBuilder;
+import org.quartz.JobDataMap;
+import org.quartz.JobDetail;
+import org.quartz.JobExecutionContext;
+import org.quartz.SchedulerException;
+import org.quartz.Trigger;
+import org.quartz.TriggerBuilder;
+
+import com.typesafe.config.Config;
+
+import javax.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.gobblin.configuration.ConfigurationKeys;
+import org.apache.gobblin.runtime.api.DagActionStore;
+import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter;
+
+
+/**
+ * Decorator used to coordinate multiple hosts with execution components
enabled to respond to flow action events with
+ * added capabilities to properly handle the result of attempted ownership
over these flow action events. It uses the
+ * {@link MultiActiveLeaseArbiter} to determine a single lease owner at a
given event time for a flow action event.
+ * After acquiring the lease, the host can pursue executing the action. Once
it has completed this action, it
+ * marks the lease as completed by calling the
+ * {@link
MultiActiveLeaseArbiter#recordLeaseSuccess(MultiActiveLeaseArbiter.LeaseObtainedStatus)}
method. Hosts
+ * that fail to acquire a lease will use the {@link
DagActionReminderScheduler} to set a reminder for the flow action
+ * event to check back in on the previous lease owner's completion status.
+ */
+@Slf4j
+public class ReminderSettingDagProcLeaseArbiter implements
MultiActiveLeaseArbiter {
+ private final MultiActiveLeaseArbiter decoratedLeaseArbiter;
+ private final DagActionReminderScheduler _dagActionReminderScheduler;
+ private final Config config;
+
+ @Inject
+ public ReminderSettingDagProcLeaseArbiter(Config config,
MultiActiveLeaseArbiter leaseArbiter, DagActionReminderScheduler
dagActionReminderScheduler) throws IOException {
+ this.decoratedLeaseArbiter = leaseArbiter;
+ this._dagActionReminderScheduler = dagActionReminderScheduler;
+ this.config = config;
+ }
+
+ /**
+ * This method is used by the multi-active scheduler and multi-active
execution classes (DagTaskStream) to attempt a
+ * lease for a particular job event and return the status of the attempt.
+ * @param flowAction
+ * @param eventTimeMillis
+ * @param isReminderEvent
+ * @param skipFlowExecutionIdReplacement
+ * @return
+ */
Review Comment:
suggest also to cut this
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/Orchestrator.java:
##########
@@ -230,15 +230,16 @@ public void orchestrate(Spec spec, Properties jobProps,
long triggerTimestampMil
Map<String, String> flowMetadata =
TimingEventUtils.getFlowMetadata(flowSpec);
String flowExecutionId =
String.valueOf(FlowUtils.getOrCreateFlowExecutionId(flowSpec));
+ // Note we use "" as jobName for lease arbitration since orchestration
is at a flow level
DagActionStore.DagAction flowAction =
- new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId,
DagActionStore.FlowActionType.LAUNCH);
+ new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId,
"", DagActionStore.DagActionType.LAUNCH);
// If multi-active scheduler is enabled do not pass onto DagManager,
otherwise scheduler forwards it directly
// Skip flow compilation as well, since we recompile after receiving
event from DagActionStoreChangeMonitor later
- if (flowTriggerHandler.isPresent()) {
+ if (flowTriggerDecorator.isPresent()) {
// Adopt consensus flowExecutionId for scheduled flows
- flowTriggerHandler.get().handleTriggerEvent(jobProps, flowAction,
triggerTimestampMillis, isReminderEvent,
+ flowTriggerDecorator.get().handleTriggerEvent(jobProps, flowAction,
triggerTimestampMillis, isReminderEvent,
Review Comment:
naming-wise it doesn't really seem like it's "handling trigger events". if
anything that's what the calling `Orchestrator::orchestrate` is doing. what
about `handleFlowLaunchTrigger`?
(post-reminders, we can't any longer treat "trigger" as a synonym for "flow
launch".)
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/Orchestrator.java:
##########
@@ -106,22 +106,22 @@ public class Orchestrator implements SpecCatalogListener,
Instrumentable {
private UserQuotaManager quotaManager;
private final FlowCompilationValidationHelper
flowCompilationValidationHelper;
- private Optional<FlowTriggerHandler> flowTriggerHandler;
+ private Optional<ReminderSettingFlowTriggerLeaseArbiter>
flowTriggerDecorator;
private Optional<FlowCatalog> flowCatalog;
@Getter
private final SharedFlowMetricsSingleton sharedFlowMetricsSingleton;
@Inject
public Orchestrator(Config config, TopologyCatalog topologyCatalog,
DagManager dagManager,
- Optional<Logger> log, FlowStatusGenerator flowStatusGenerator,
Optional<FlowTriggerHandler> flowTriggerHandler,
+ Optional<Logger> log, FlowStatusGenerator flowStatusGenerator,
Optional<ReminderSettingFlowTriggerLeaseArbiter> flowTriggerDecorator,
Review Comment:
the fact that we can't indicate the type here as `MultiActiveLeaseArbiter`
is another clue that this type is not at heart a Decorator, but instead
something that must `handleTriggerEvent`
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/Orchestrator.java:
##########
@@ -230,15 +230,16 @@ public void orchestrate(Spec spec, Properties jobProps,
long triggerTimestampMil
Map<String, String> flowMetadata =
TimingEventUtils.getFlowMetadata(flowSpec);
String flowExecutionId =
String.valueOf(FlowUtils.getOrCreateFlowExecutionId(flowSpec));
+ // Note we use "" as jobName for lease arbitration since orchestration
is at a flow level
DagActionStore.DagAction flowAction =
Review Comment:
nit: `launchDagAction`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]