phet commented on code in PR #3878:
URL: https://github.com/apache/gobblin/pull/3878#discussion_r1490500328
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagStateStore.java:
##########
@@ -70,4 +71,8 @@ public interface DagStateStore {
* Return a list of all dag IDs contained in the dag state store.
*/
Set<String> getDagIds() throws IOException;
+
+ default boolean existsDag(String dagId) throws IOException {
+ throw new UnsupportedEncodingException("containsDag not implemented");
Review Comment:
NBD, but I believe you want `java.lang.UnsupportedOperationException`
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/MostlyInMemoryDagManagementStateStore.java:
##########
@@ -130,44 +122,42 @@ public Set<String> getFailedDagIds() throws IOException {
}
@Override
- public synchronized void deleteDagNodeState(String dagId,
Dag.DagNode<JobExecutionPlan> dagNode) {
+ public synchronized void deleteDagNodeState(DagManager.DagId dagId,
Dag.DagNode<JobExecutionPlan> dagNode) {
+ String dagIdStr = dagId.toString();
this.jobToDag.remove(dagNode);
this.dagNodes.remove(dagNode.getValue().getId());
- this.dagToDeadline.remove(dagId);
- this.dagToJobs.get(dagId).remove(dagNode);
- if (this.dagToJobs.get(dagId).isEmpty()) {
- this.dagToJobs.remove(dagId);
+ this.dagToDeadline.remove(dagIdStr);
+ this.dagToJobs.get(dagIdStr).remove(dagNode);
+ if (this.dagToJobs.get(dagIdStr).isEmpty()) {
+ this.dagToJobs.remove(dagIdStr);
}
}
@Override
- public synchronized void addDagNodeState(String dagId,
Dag.DagNode<JobExecutionPlan> dagNode) {
- Dag<JobExecutionPlan> dag = this.dags.get(dagId);
+ public synchronized void addDagNodeState(DagManager.DagId dagId,
Dag.DagNode<JobExecutionPlan> dagNode)
+ throws IOException {
+ String dagIdStr = dagId.toString();
+ Dag<JobExecutionPlan> dag = getDag(dagId);
this.jobToDag.put(dagNode, dag);
this.dagNodes.put(dagNode.getValue().getId(), dagNode);
- if (!this.dagToJobs.containsKey(dagId)) {
- this.dagToJobs.put(dagId, Lists.newLinkedList());
+ if (!this.dagToJobs.containsKey(dagIdStr)) {
+ this.dagToJobs.put(dagIdStr, Lists.newLinkedList());
}
- this.dagToJobs.get(dagId).add(dagNode);
- }
-
- @Override
- public Dag<JobExecutionPlan> getDag(DagManager.DagId dagId) {
- return this.dags.get(dagId.toString());
+ this.dagToJobs.get(dagIdStr).add(dagNode);
}
@Override
- public void addDag(Dag<JobExecutionPlan> dag) {
- this.dags.put(DagManagerUtils.generateDagId(dag).toString(), dag);
+ public Dag<JobExecutionPlan> getDag(DagManager.DagId dagId) throws
IOException {
+ return this.dagStateStore.getDag(dagId.toString());
}
@Override
- public boolean containsDag(String dagId) {
- return this.dags.containsKey(dagId);
+ public boolean containsDag(DagManager.DagId dagId) throws IOException {
+ return this.dagStateStore.existsDag(dagId.toString());
Review Comment:
`DagId` seems the better param type for the `DagStateStore` methods than
`String`. I suggest we move to that by making default methods in the DSS
interface that call `.toString()` there, in the interface itself, rather than
here
then either as part of this PR or leave a TODO to deprecate the `String
dagId` forms
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/MostlyInMemoryDagManagementStateStore.java:
##########
@@ -130,44 +122,42 @@ public Set<String> getFailedDagIds() throws IOException {
}
@Override
- public synchronized void deleteDagNodeState(String dagId,
Dag.DagNode<JobExecutionPlan> dagNode) {
+ public synchronized void deleteDagNodeState(DagManager.DagId dagId,
Dag.DagNode<JobExecutionPlan> dagNode) {
+ String dagIdStr = dagId.toString();
this.jobToDag.remove(dagNode);
this.dagNodes.remove(dagNode.getValue().getId());
- this.dagToDeadline.remove(dagId);
- this.dagToJobs.get(dagId).remove(dagNode);
- if (this.dagToJobs.get(dagId).isEmpty()) {
- this.dagToJobs.remove(dagId);
+ this.dagToDeadline.remove(dagIdStr);
+ this.dagToJobs.get(dagIdStr).remove(dagNode);
+ if (this.dagToJobs.get(dagIdStr).isEmpty()) {
+ this.dagToJobs.remove(dagIdStr);
}
}
@Override
- public synchronized void addDagNodeState(String dagId,
Dag.DagNode<JobExecutionPlan> dagNode) {
- Dag<JobExecutionPlan> dag = this.dags.get(dagId);
+ public synchronized void addDagNodeState(DagManager.DagId dagId,
Dag.DagNode<JobExecutionPlan> dagNode)
+ throws IOException {
+ String dagIdStr = dagId.toString();
+ Dag<JobExecutionPlan> dag = getDag(dagId);
this.jobToDag.put(dagNode, dag);
this.dagNodes.put(dagNode.getValue().getId(), dagNode);
- if (!this.dagToJobs.containsKey(dagId)) {
- this.dagToJobs.put(dagId, Lists.newLinkedList());
+ if (!this.dagToJobs.containsKey(dagIdStr)) {
+ this.dagToJobs.put(dagIdStr, Lists.newLinkedList());
}
- this.dagToJobs.get(dagId).add(dagNode);
- }
-
- @Override
- public Dag<JobExecutionPlan> getDag(DagManager.DagId dagId) {
- return this.dags.get(dagId.toString());
+ this.dagToJobs.get(dagIdStr).add(dagNode);
Review Comment:
I didn't see your response to my earlier comment...
given DagId has equals and hash code, could we make it the key type, rather
than String?
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/MysqlDagStateStore.java:
##########
@@ -134,6 +134,11 @@ public void cleanUp(String dagId)
this.totalDagCount.dec();
}
+ @Override
+ public boolean existsDag(String dagId) throws IOException {
Review Comment:
should this param type be `DagId`, not `String`?
##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManagementStateStore.java:
##########
@@ -32,44 +33,49 @@
* and allows add/delete and other functions.
*/
public interface DagManagementStateStore {
- void addDag(Dag<JobExecutionPlan> dag);
/**
* Persist the {@link Dag} to the backing store.
- * This is not an actual checkpoint but more like a Write-ahead log, where
uncommitted job will be persisted
- * and be picked up again when leader transition happens.
+ * It is named this because it is usually called to checkpoint any changes
in {@link Dag} or in its {@link Dag.DagNode}s.
* @param dag The dag submitted to {@link DagManager}
*/
void writeCheckpoint(Dag<JobExecutionPlan> dag) throws IOException;
Review Comment:
seems like this has been combined w/ `addDag` (which is great!)... but why
not keep that name rather than this one? `addDag` seems clearer and also
aligns better w/ `containsDag` and `getDag`.
relatedly could we rename `cleanUp` to `deleteDag`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]