umustafi commented on code in PR #3890: URL: https://github.com/apache/gobblin/pull/3890#discussion_r1524053966
########## gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagProcReminderScheduler.java: ########## @@ -0,0 +1,47 @@ +package org.apache.gobblin.service.modules.orchestration; + +import org.quartz.*; +import org.quartz.impl.StdSchedulerFactory; Review Comment: I created an instance of `org.quartz.impl.StdSchedulerFactory` in Guice to re-use across JVM which is why I'm passing the reference here and using it to create a `Scheduler` (we don't need the `SchedulerService` interface capabilities necessarily to make it). Do you have other suggestions in mind? ########## gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/InstrumentedLeaseArbiterDecorator.java: ########## @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.service.modules.orchestration; + +import java.io.IOException; + +import com.typesafe.config.Config; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.gobblin.instrumented.Instrumented; +import org.apache.gobblin.metrics.ContextAwareCounter; +import org.apache.gobblin.metrics.ContextAwareMeter; +import org.apache.gobblin.metrics.MetricContext; +import org.apache.gobblin.metrics.ServiceMetricNames; +import org.apache.gobblin.runtime.api.DagActionStore; +import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter; +import org.apache.gobblin.runtime.api.MysqlMultiActiveLeaseArbiter; +import org.apache.gobblin.util.ConfigUtils; + + +/* + A generic lease arbitration decorator built upon the {@link MysqlMultiActiveLeaseArbiter} which encapsulates common + functionality desired by lease arbiter users to track metrics on all lease attempts made by the arbiter. The metrics + can be used to compare relative performance among arbitration participants. + */ +@Slf4j +public abstract class InstrumentedLeaseArbiterDecorator extends MysqlMultiActiveLeaseArbiter { Review Comment: Updated PR to follow this pattern. ########## gobblin-runtime/src/main/java/org/apache/gobblin/scheduler/SchedulerService.java: ########## @@ -60,20 +61,27 @@ public SchedulerService(Properties props) { Optional.of(PropertiesUtils.extractPropertiesWithPrefix(props, Optional.of("org.quartz.")))); } - @Inject public SchedulerService(Config cfg) { this(cfg.hasPath(ConfigurationKeys.SCHEDULER_WAIT_FOR_JOB_COMPLETION_KEY) ? cfg.getBoolean(ConfigurationKeys.SCHEDULER_WAIT_FOR_JOB_COMPLETION_KEY) : Boolean.parseBoolean(ConfigurationKeys.DEFAULT_SCHEDULER_WAIT_FOR_JOB_COMPLETION), Optional.of(ConfigUtils.configToProperties(cfg, "org.quartz."))); } + @Inject + public SchedulerService(Config cfg, StdSchedulerFactory stdSchedulerFactory) { + this(cfg); + this.stdSchedulerFactory = stdSchedulerFactory; + } + @Override protected void startUp() throws SchedulerException { - StdSchedulerFactory schedulerFactory = new StdSchedulerFactory(); + if (this.stdSchedulerFactory == null) { + this.stdSchedulerFactory = new StdSchedulerFactory(); + } Review Comment: I moved the intialization of the stdSchedulerFactory to the lowest level constructor to always initialize the factory if one is not provided. ########## gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/InstrumentedLeaseArbiterDecorator.java: ########## @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.service.modules.orchestration; + +import java.io.IOException; + +import com.typesafe.config.Config; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.gobblin.instrumented.Instrumented; +import org.apache.gobblin.metrics.ContextAwareCounter; +import org.apache.gobblin.metrics.ContextAwareMeter; +import org.apache.gobblin.metrics.MetricContext; +import org.apache.gobblin.metrics.ServiceMetricNames; +import org.apache.gobblin.runtime.api.DagActionStore; +import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter; +import org.apache.gobblin.runtime.api.MysqlMultiActiveLeaseArbiter; +import org.apache.gobblin.util.ConfigUtils; + + +/* + A generic lease arbitration decorator built upon the {@link MysqlMultiActiveLeaseArbiter} which encapsulates common + functionality desired by lease arbiter users to track metrics on all lease attempts made by the arbiter. The metrics + can be used to compare relative performance among arbitration participants. + */ +@Slf4j +public abstract class InstrumentedLeaseArbiterDecorator extends MysqlMultiActiveLeaseArbiter { + protected MultiActiveLeaseArbiter multiActiveLeaseArbiter; + protected MetricContext metricContext; + private ContextAwareCounter leaseObtainedCount; + + private ContextAwareCounter leasedToAnotherStatusCount; + + private ContextAwareCounter noLongerLeasingStatusCount; + private ContextAwareMeter leasesObtainedDueToReminderCount; + + public InstrumentedLeaseArbiterDecorator(Config config, MultiActiveLeaseArbiter leaseDeterminationStore, + String metricsPrefix) throws IOException { + super(config); + this.multiActiveLeaseArbiter = leaseDeterminationStore; + this.metricContext = Instrumented.getMetricContext(new org.apache.gobblin.configuration.State(ConfigUtils.configToProperties(config)), + this.getClass()); + initializeMetrics(metricsPrefix); + } + + private void initializeMetrics(String metricsPrefix) { + // If a valid metrics prefix is provided then add a delimiter after it + if (metricsPrefix != "") { + metricsPrefix += "."; + } + this.leaseObtainedCount = this.metricContext.contextAwareCounter(metricsPrefix + ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASE_OBTAINED_COUNT); + this.leasedToAnotherStatusCount = this.metricContext.contextAwareCounter(metricsPrefix + ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASED_TO_ANOTHER_COUNT); + this.noLongerLeasingStatusCount = this.metricContext.contextAwareCounter(metricsPrefix + ServiceMetricNames.FLOW_TRIGGER_HANDLER_NO_LONGER_LEASING_COUNT); + this.leasesObtainedDueToReminderCount = this.metricContext.contextAwareMeter(metricsPrefix + ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASES_OBTAINED_DUE_TO_REMINDER_COUNT); + } + + /** + * This method is used by the multi-active scheduler and multi-active execution classes (DagTaskStream) to attempt a + * lease for a particular job event and return the status of the attempt. + * @param flowAction + * @param eventTimeMillis + * @param isReminderEvent + * @param skipFlowExecutionIdReplacement + * @return + */ + @Override + public MultiActiveLeaseArbiter.LeaseAttemptStatus tryAcquireLease(DagActionStore.DagAction flowAction, long eventTimeMillis, + boolean isReminderEvent, boolean skipFlowExecutionIdReplacement) throws IOException { + + MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus = + multiActiveLeaseArbiter.tryAcquireLease(flowAction, eventTimeMillis, isReminderEvent, + skipFlowExecutionIdReplacement); + // The flow action contained in the`LeaseAttemptStatus` from the lease arbiter contains an updated flow execution + // id. From this point onwards, always use the newer version of the flow action to easily track the action through + // orchestration and execution. + if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.LeaseObtainedStatus) { + if (isReminderEvent) { + this.leasesObtainedDueToReminderCount.mark(); + } + this.leaseObtainedCount.inc(); + return leaseAttemptStatus; + } else if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.LeasedToAnotherStatus) { + this.leasedToAnotherStatusCount.inc(); +// scheduleReminderForEvent((MultiActiveLeaseArbiter.LeasedToAnotherStatus) leaseAttemptStatus, eventTimeMillis); + return leaseAttemptStatus; + } else if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.NoLongerLeasingStatus) { + this.noLongerLeasingStatusCount.inc(); + log.debug("Received type of leaseAttemptStatus: [{}, eventTimestamp: {}] ", leaseAttemptStatus.getClass().getName(), Review Comment: Moved to info level for all cases of lease attempt status ########## gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/InstrumentedLeaseArbiterDecorator.java: ########## @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.service.modules.orchestration; + +import java.io.IOException; + +import com.typesafe.config.Config; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.gobblin.instrumented.Instrumented; +import org.apache.gobblin.metrics.ContextAwareCounter; +import org.apache.gobblin.metrics.ContextAwareMeter; +import org.apache.gobblin.metrics.MetricContext; +import org.apache.gobblin.metrics.ServiceMetricNames; +import org.apache.gobblin.runtime.api.DagActionStore; +import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter; +import org.apache.gobblin.runtime.api.MysqlMultiActiveLeaseArbiter; +import org.apache.gobblin.util.ConfigUtils; + + +/* + A generic lease arbitration decorator built upon the {@link MysqlMultiActiveLeaseArbiter} which encapsulates common + functionality desired by lease arbiter users to track metrics on all lease attempts made by the arbiter. The metrics + can be used to compare relative performance among arbitration participants. + */ +@Slf4j +public abstract class InstrumentedLeaseArbiterDecorator extends MysqlMultiActiveLeaseArbiter { + protected MultiActiveLeaseArbiter multiActiveLeaseArbiter; + protected MetricContext metricContext; + private ContextAwareCounter leaseObtainedCount; + + private ContextAwareCounter leasedToAnotherStatusCount; + + private ContextAwareCounter noLongerLeasingStatusCount; + private ContextAwareMeter leasesObtainedDueToReminderCount; + + public InstrumentedLeaseArbiterDecorator(Config config, MultiActiveLeaseArbiter leaseDeterminationStore, + String metricsPrefix) throws IOException { + super(config); + this.multiActiveLeaseArbiter = leaseDeterminationStore; + this.metricContext = Instrumented.getMetricContext(new org.apache.gobblin.configuration.State(ConfigUtils.configToProperties(config)), + this.getClass()); + initializeMetrics(metricsPrefix); + } + + private void initializeMetrics(String metricsPrefix) { + // If a valid metrics prefix is provided then add a delimiter after it + if (metricsPrefix != "") { Review Comment: It should have a class name ideally but I don't think it makes sense to throw an error if one isn't provided. It can still use the metrics names without a prefix. We can log a warning if we want. ########## gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/Orchestrator.java: ########## @@ -230,15 +230,16 @@ public void orchestrate(Spec spec, Properties jobProps, long triggerTimestampMil Map<String, String> flowMetadata = TimingEventUtils.getFlowMetadata(flowSpec); String flowExecutionId = String.valueOf(FlowUtils.getOrCreateFlowExecutionId(flowSpec)); + // Note we use flowName as jobName for lease arbitration since orchestration is at a flow level Review Comment: this was decided later on, using "" ########## gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/Orchestrator.java: ########## @@ -230,15 +230,16 @@ public void orchestrate(Spec spec, Properties jobProps, long triggerTimestampMil Map<String, String> flowMetadata = TimingEventUtils.getFlowMetadata(flowSpec); String flowExecutionId = String.valueOf(FlowUtils.getOrCreateFlowExecutionId(flowSpec)); + // Note we use flowName as jobName for lease arbitration since orchestration is at a flow level DagActionStore.DagAction flowAction = - new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId, DagActionStore.FlowActionType.LAUNCH); + new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId, flowName, DagActionStore.FlowActionType.LAUNCH); Review Comment: Updated to reflect most recent agreement of `""` no jobName applicable ########## gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagProcArbiterDecorator.java: ########## @@ -0,0 +1,140 @@ +package org.apache.gobblin.service.modules.orchestration; + + +import java.io.IOException; +import java.util.Date; + +import org.quartz.Job; +import org.quartz.JobBuilder; +import org.quartz.JobDataMap; +import org.quartz.JobDetail; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.quartz.SchedulerException; +import org.quartz.Trigger; +import org.quartz.TriggerBuilder; + +import com.google.common.base.Optional; +import com.typesafe.config.Config; + +import javax.inject.Inject; +import lombok.extern.slf4j.Slf4j; + +import org.apache.gobblin.configuration.ConfigurationKeys; +import org.apache.gobblin.runtime.api.DagActionStore; +import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter; +import org.apache.gobblin.runtime.api.MysqlMultiActiveLeaseArbiter; + + +/** + * Decorator used to coordinate multiple hosts with execution components enabled to respond to flow action events. It + * uses the {@link MultiActiveLeaseArbiter} to determine a single lease owner at a given event time for a flow action + * event. After acquiring the lease, the host can pursue executing the action. Once it has completed this action, it + * marks the lease as completed by calling the + * {@link MysqlMultiActiveLeaseArbiter#recordLeaseSuccess(MultiActiveLeaseArbiter.LeaseObtainedStatus)} method. Hosts + * that fail to acquire a lease will use the {@link DagProcReminderScheduler} to set a reminder for the flow action + * event to check back in on the previous lease owner's completion status. + */ +public class DagProcArbiterDecorator extends InstrumentedLeaseArbiterDecorator { Review Comment: I updated the implementation. Let me know your thoughts. ########## gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerDecorator.java: ########## @@ -64,43 +62,29 @@ * in failure cases. */ @Slf4j -public class FlowTriggerHandler { +public class FlowTriggerDecorator extends InstrumentedLeaseArbiterDecorator { Review Comment: Both `DagProc` and `FlowTrigger` decorator's are self-healing (set own reminders), but they're specific implementations for different use cases with different implementations. The naming could have suffix `[DagProc/FlowTrigger]LeaseArbiterDecoratorWith[Reminder/Self-Healing]` but it feels like a mouthful. I will update both Java docs to describe the additional functionality which may be sufficient. ########## gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagProcArbiterDecorator.java: ########## @@ -0,0 +1,140 @@ +package org.apache.gobblin.service.modules.orchestration; + + +import java.io.IOException; +import java.util.Date; + +import org.quartz.Job; +import org.quartz.JobBuilder; +import org.quartz.JobDataMap; +import org.quartz.JobDetail; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.quartz.SchedulerException; +import org.quartz.Trigger; +import org.quartz.TriggerBuilder; + +import com.google.common.base.Optional; +import com.typesafe.config.Config; + +import javax.inject.Inject; +import lombok.extern.slf4j.Slf4j; + +import org.apache.gobblin.configuration.ConfigurationKeys; +import org.apache.gobblin.runtime.api.DagActionStore; +import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter; +import org.apache.gobblin.runtime.api.MysqlMultiActiveLeaseArbiter; + + +/** + * Decorator used to coordinate multiple hosts with execution components enabled to respond to flow action events. It + * uses the {@link MultiActiveLeaseArbiter} to determine a single lease owner at a given event time for a flow action + * event. After acquiring the lease, the host can pursue executing the action. Once it has completed this action, it + * marks the lease as completed by calling the + * {@link MysqlMultiActiveLeaseArbiter#recordLeaseSuccess(MultiActiveLeaseArbiter.LeaseObtainedStatus)} method. Hosts + * that fail to acquire a lease will use the {@link DagProcReminderScheduler} to set a reminder for the flow action + * event to check back in on the previous lease owner's completion status. + */ +public class DagProcArbiterDecorator extends InstrumentedLeaseArbiterDecorator { + private final DagProcReminderScheduler dagProcReminderScheduler; + + @Inject + public DagProcArbiterDecorator(Config config, Optional<MultiActiveLeaseArbiter> leaseDeterminationStore, + DagProcReminderScheduler dagProcReminderScheduler) throws IOException { + super(config, leaseDeterminationStore.get(), String.valueOf(DagProcArbiterDecorator.class)); + this.dagProcReminderScheduler = dagProcReminderScheduler; + } + + /** + * This method is used by the multi-active scheduler and multi-active execution classes (DagTaskStream) to attempt a + * lease for a particular job event and return the status of the attempt. + * @param flowAction + * @param eventTimeMillis + * @param isReminderEvent + * @param skipFlowExecutionIdReplacement + * @return + */ + @Override + public MultiActiveLeaseArbiter.LeaseAttemptStatus tryAcquireLease(DagActionStore.DagAction flowAction, long eventTimeMillis, + boolean isReminderEvent, boolean skipFlowExecutionIdReplacement) { + try { + MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus = this.tryAcquireLease(flowAction, eventTimeMillis, isReminderEvent, skipFlowExecutionIdReplacement); + if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.LeasedToAnotherStatus) { + scheduleReminderForEvent((MultiActiveLeaseArbiter.LeasedToAnotherStatus) leaseAttemptStatus, eventTimeMillis); + return leaseAttemptStatus; + } + /* Otherwise leaseAttemptStatus instanceof {@link MultiActiveLeaseArbiter.LeaseObtainedStatus/NoLongerLeasingStatus} + and no need to do anything + */ + return leaseAttemptStatus; + } catch (SchedulerException e) { + throw new RuntimeException(e); + } + } + + protected void scheduleReminderForEvent(MultiActiveLeaseArbiter.LeasedToAnotherStatus leaseStatus, long triggerEventTimeMillis) + throws SchedulerException { + // TODO: determine which ts to use + dagProcReminderScheduler.scheduleReminderJob(leaseStatus.getFlowAction(), leaseStatus.getEventTimeMillis()); + + } + + @Slf4j + public class ReminderJob implements Job { + public static final String FLOW_ACTION_TYPE_KEY = "flow.actionType"; + public static final String DAG_TASK_STREAM = "dag.taskStream"; + + @Override + public void execute(JobExecutionContext context) throws JobExecutionException { + // Get properties from the trigger to create a dagAction + String flowName = context.getTrigger().getJobDataMap().getString(ConfigurationKeys.FLOW_NAME_KEY); + String flowGroup = context.getTrigger().getJobDataMap().getString(ConfigurationKeys.FLOW_GROUP_KEY); + String jobName = context.getTrigger().getJobDataMap().getString(ConfigurationKeys.JOB_NAME_KEY); + String flowId = context.getTrigger().getJobDataMap().getString(ConfigurationKeys.FLOW_EXECUTION_ID_KEY); + DagActionStore.FlowActionType flowActionType = DagActionStore.FlowActionType.valueOf( + context.getTrigger().getJobDataMap().getString(FLOW_ACTION_TYPE_KEY)); + DagManagementTaskStreamImpl dagManagementTaskStream = + (DagManagementTaskStreamImpl) context.getTrigger().getJobDataMap().get(DAG_TASK_STREAM); + + // TODO: add meaningful log statement + log.info("Reminder for job " + jobName + " in flow " + flowName + " (" + flowGroup + ") with ID " + flowId); + + DagActionStore.DagAction dagAction = new DagActionStore.DagAction(flowGroup, flowName, flowId, jobName, flowActionType); + + dagManagementTaskStream.addDagAction(dagAction); + } + } + + public static String createDagActionReminderKey(DagActionStore.DagAction dagAction) { + return createDagActionReminderKey(dagAction.getFlowName(), dagAction.getFlowGroup(), dagAction.getJobName(), + dagAction.getFlowExecutionId(), dagAction.getFlowActionType()); + } + + public static String createDagActionReminderKey(String flowName, String flowGroup, String jobName, String flowId, + DagActionStore.FlowActionType flowActionType) { + return String.format("%s.%s.%s.%s.%s", flowName, flowGroup, jobName, flowId, flowActionType); + } + + public static JobDetail createReminderJobDetail(DagManagementTaskStreamImpl taskStream, DagActionStore.DagAction dagAction) { + JobDataMap dataMap = new JobDataMap(); + dataMap.put(ReminderJob.DAG_TASK_STREAM, taskStream); Review Comment: Updated to be `DagManagement` btw. I needed a reference to it from `ReminderJob` to re-add the action to be evaluated once lease should expire. `ReminderJob` should be a static class like other job types so I opted to pass the reference here although I agree it's less than ideal. Do you prefer making `ReminderJob` non static and contain a reference to the `DagManagement` instance? ########## gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagProcArbiterDecorator.java: ########## @@ -0,0 +1,140 @@ +package org.apache.gobblin.service.modules.orchestration; + + +import java.io.IOException; +import java.util.Date; + +import org.quartz.Job; +import org.quartz.JobBuilder; +import org.quartz.JobDataMap; +import org.quartz.JobDetail; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.quartz.SchedulerException; +import org.quartz.Trigger; +import org.quartz.TriggerBuilder; + +import com.google.common.base.Optional; +import com.typesafe.config.Config; + +import javax.inject.Inject; +import lombok.extern.slf4j.Slf4j; + +import org.apache.gobblin.configuration.ConfigurationKeys; +import org.apache.gobblin.runtime.api.DagActionStore; +import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter; +import org.apache.gobblin.runtime.api.MysqlMultiActiveLeaseArbiter; + + +/** + * Decorator used to coordinate multiple hosts with execution components enabled to respond to flow action events. It + * uses the {@link MultiActiveLeaseArbiter} to determine a single lease owner at a given event time for a flow action + * event. After acquiring the lease, the host can pursue executing the action. Once it has completed this action, it + * marks the lease as completed by calling the + * {@link MysqlMultiActiveLeaseArbiter#recordLeaseSuccess(MultiActiveLeaseArbiter.LeaseObtainedStatus)} method. Hosts + * that fail to acquire a lease will use the {@link DagProcReminderScheduler} to set a reminder for the flow action + * event to check back in on the previous lease owner's completion status. + */ +public class DagProcArbiterDecorator extends InstrumentedLeaseArbiterDecorator { + private final DagProcReminderScheduler dagProcReminderScheduler; + + @Inject + public DagProcArbiterDecorator(Config config, Optional<MultiActiveLeaseArbiter> leaseDeterminationStore, + DagProcReminderScheduler dagProcReminderScheduler) throws IOException { + super(config, leaseDeterminationStore.get(), String.valueOf(DagProcArbiterDecorator.class)); + this.dagProcReminderScheduler = dagProcReminderScheduler; + } + + /** + * This method is used by the multi-active scheduler and multi-active execution classes (DagTaskStream) to attempt a + * lease for a particular job event and return the status of the attempt. + * @param flowAction + * @param eventTimeMillis + * @param isReminderEvent + * @param skipFlowExecutionIdReplacement + * @return + */ + @Override + public MultiActiveLeaseArbiter.LeaseAttemptStatus tryAcquireLease(DagActionStore.DagAction flowAction, long eventTimeMillis, + boolean isReminderEvent, boolean skipFlowExecutionIdReplacement) { + try { + MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus = this.tryAcquireLease(flowAction, eventTimeMillis, isReminderEvent, skipFlowExecutionIdReplacement); + if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.LeasedToAnotherStatus) { + scheduleReminderForEvent((MultiActiveLeaseArbiter.LeasedToAnotherStatus) leaseAttemptStatus, eventTimeMillis); + return leaseAttemptStatus; + } + /* Otherwise leaseAttemptStatus instanceof {@link MultiActiveLeaseArbiter.LeaseObtainedStatus/NoLongerLeasingStatus} + and no need to do anything + */ + return leaseAttemptStatus; Review Comment: I agree with the rationale and am setting a reminder in both cases now. See overloaded method for setting reminder, one taking `LeaseObtainedStatus` and another `LeasedToAnotherStatus` as params ########## gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerDecorator.java: ########## @@ -116,54 +100,41 @@ public FlowTriggerHandler(Config config, Optional<MultiActiveLeaseArbiter> lease * @throws IOException */ public void handleTriggerEvent(Properties jobProps, DagActionStore.DagAction flowAction, long eventTimeMillis, - boolean isReminderEvent, boolean skipFlowExecutionIdReplacement) throws IOException { - if (multiActiveLeaseArbiter.isPresent()) { - MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus = multiActiveLeaseArbiter.get().tryAcquireLease( - flowAction, eventTimeMillis, isReminderEvent, skipFlowExecutionIdReplacement); - // The flow action contained in the`LeaseAttemptStatus` from the lease arbiter contains an updated flow execution - // id. From this point onwards, always use the newer version of the flow action to easily track the action through - // orchestration and execution. + boolean isReminderEvent, boolean skipFlowExecutionIdReplacement) + throws IOException { + + try { + LeaseAttemptStatus leaseAttemptStatus = null; + leaseAttemptStatus = + this.tryAcquireLease(flowAction, eventTimeMillis, isReminderEvent, skipFlowExecutionIdReplacement); if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.LeaseObtainedStatus) { - if (isReminderEvent) { - this.leasesObtainedDueToReminderCount.mark(); + MultiActiveLeaseArbiter.LeaseObtainedStatus leaseObtainedStatus = (MultiActiveLeaseArbiter.LeaseObtainedStatus) + leaseAttemptStatus; + if (persistFlowAction(leaseObtainedStatus)) { Review Comment: We used `flowAction` to describe actions operating at a `flow-level` (instead of `dag`), but it's been confusing because the type of the object is always `DagAction` and now are broadening this type to have Flow and Job level actions. I am unifying around the naming of `DagAction` and have the flow/job properties determine the scope of the action. ########## gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/InstrumentedLeaseArbiterDecorator.java: ########## @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.service.modules.orchestration; + +import java.io.IOException; + +import com.typesafe.config.Config; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.gobblin.instrumented.Instrumented; +import org.apache.gobblin.metrics.ContextAwareCounter; +import org.apache.gobblin.metrics.ContextAwareMeter; +import org.apache.gobblin.metrics.MetricContext; +import org.apache.gobblin.metrics.ServiceMetricNames; +import org.apache.gobblin.runtime.api.DagActionStore; +import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter; +import org.apache.gobblin.runtime.api.MysqlMultiActiveLeaseArbiter; +import org.apache.gobblin.util.ConfigUtils; + + +/* + A generic lease arbitration decorator built upon the {@link MysqlMultiActiveLeaseArbiter} which encapsulates common + functionality desired by lease arbiter users to track metrics on all lease attempts made by the arbiter. The metrics + can be used to compare relative performance among arbitration participants. + */ +@Slf4j +public abstract class InstrumentedLeaseArbiterDecorator extends MysqlMultiActiveLeaseArbiter { + protected MultiActiveLeaseArbiter multiActiveLeaseArbiter; + protected MetricContext metricContext; + private ContextAwareCounter leaseObtainedCount; + + private ContextAwareCounter leasedToAnotherStatusCount; + + private ContextAwareCounter noLongerLeasingStatusCount; + private ContextAwareMeter leasesObtainedDueToReminderCount; + + public InstrumentedLeaseArbiterDecorator(Config config, MultiActiveLeaseArbiter leaseDeterminationStore, + String metricsPrefix) throws IOException { + super(config); + this.multiActiveLeaseArbiter = leaseDeterminationStore; + this.metricContext = Instrumented.getMetricContext(new org.apache.gobblin.configuration.State(ConfigUtils.configToProperties(config)), + this.getClass()); + initializeMetrics(metricsPrefix); + } + + private void initializeMetrics(String metricsPrefix) { + // If a valid metrics prefix is provided then add a delimiter after it + if (metricsPrefix != "") { + metricsPrefix += "."; + } + this.leaseObtainedCount = this.metricContext.contextAwareCounter(metricsPrefix + ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASE_OBTAINED_COUNT); + this.leasedToAnotherStatusCount = this.metricContext.contextAwareCounter(metricsPrefix + ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASED_TO_ANOTHER_COUNT); + this.noLongerLeasingStatusCount = this.metricContext.contextAwareCounter(metricsPrefix + ServiceMetricNames.FLOW_TRIGGER_HANDLER_NO_LONGER_LEASING_COUNT); + this.leasesObtainedDueToReminderCount = this.metricContext.contextAwareMeter(metricsPrefix + ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASES_OBTAINED_DUE_TO_REMINDER_COUNT); + } + + /** + * This method is used by the multi-active scheduler and multi-active execution classes (DagTaskStream) to attempt a + * lease for a particular job event and return the status of the attempt. + * @param flowAction + * @param eventTimeMillis + * @param isReminderEvent + * @param skipFlowExecutionIdReplacement + * @return + */ + @Override + public MultiActiveLeaseArbiter.LeaseAttemptStatus tryAcquireLease(DagActionStore.DagAction flowAction, long eventTimeMillis, + boolean isReminderEvent, boolean skipFlowExecutionIdReplacement) throws IOException { + + MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus = + multiActiveLeaseArbiter.tryAcquireLease(flowAction, eventTimeMillis, isReminderEvent, + skipFlowExecutionIdReplacement); + // The flow action contained in the`LeaseAttemptStatus` from the lease arbiter contains an updated flow execution + // id. From this point onwards, always use the newer version of the flow action to easily track the action through + // orchestration and execution. Review Comment: I agree, moved it to the javadoc of this method. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
