phet commented on code in PR #3878:
URL: https://github.com/apache/gobblin/pull/3878#discussion_r1490500328


##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagStateStore.java:
##########
@@ -70,4 +71,8 @@ public interface DagStateStore {
    * Return a list of all dag IDs contained in the dag state store.
    */
   Set<String> getDagIds() throws IOException;
+
+  default boolean existsDag(String dagId) throws IOException {
+    throw new UnsupportedEncodingException("containsDag not implemented");

Review Comment:
   NBD, but I believe you want `java.lang.UnsupportedOperationException`



##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/MostlyInMemoryDagManagementStateStore.java:
##########
@@ -130,44 +122,42 @@ public Set<String> getFailedDagIds() throws IOException {
   }
 
   @Override
-  public synchronized void deleteDagNodeState(String dagId, 
Dag.DagNode<JobExecutionPlan> dagNode) {
+  public synchronized void deleteDagNodeState(DagManager.DagId dagId, 
Dag.DagNode<JobExecutionPlan> dagNode) {
+    String dagIdStr = dagId.toString();
     this.jobToDag.remove(dagNode);
     this.dagNodes.remove(dagNode.getValue().getId());
-    this.dagToDeadline.remove(dagId);
-    this.dagToJobs.get(dagId).remove(dagNode);
-    if (this.dagToJobs.get(dagId).isEmpty()) {
-      this.dagToJobs.remove(dagId);
+    this.dagToDeadline.remove(dagIdStr);
+    this.dagToJobs.get(dagIdStr).remove(dagNode);
+    if (this.dagToJobs.get(dagIdStr).isEmpty()) {
+      this.dagToJobs.remove(dagIdStr);
     }
   }
 
   @Override
-  public synchronized void addDagNodeState(String dagId, 
Dag.DagNode<JobExecutionPlan> dagNode) {
-    Dag<JobExecutionPlan> dag = this.dags.get(dagId);
+  public synchronized void addDagNodeState(DagManager.DagId dagId, 
Dag.DagNode<JobExecutionPlan> dagNode)
+      throws IOException {
+    String dagIdStr = dagId.toString();
+    Dag<JobExecutionPlan> dag = getDag(dagId);
     this.jobToDag.put(dagNode, dag);
     this.dagNodes.put(dagNode.getValue().getId(), dagNode);
-    if (!this.dagToJobs.containsKey(dagId)) {
-      this.dagToJobs.put(dagId, Lists.newLinkedList());
+    if (!this.dagToJobs.containsKey(dagIdStr)) {
+      this.dagToJobs.put(dagIdStr, Lists.newLinkedList());
     }
-    this.dagToJobs.get(dagId).add(dagNode);
-  }
-
-  @Override
-  public Dag<JobExecutionPlan> getDag(DagManager.DagId dagId) {
-    return this.dags.get(dagId.toString());
+    this.dagToJobs.get(dagIdStr).add(dagNode);
   }
 
   @Override
-  public void addDag(Dag<JobExecutionPlan> dag) {
-    this.dags.put(DagManagerUtils.generateDagId(dag).toString(), dag);
+  public Dag<JobExecutionPlan> getDag(DagManager.DagId dagId) throws 
IOException {
+    return this.dagStateStore.getDag(dagId.toString());
   }
 
   @Override
-  public boolean containsDag(String dagId) {
-    return this.dags.containsKey(dagId);
+  public boolean containsDag(DagManager.DagId dagId) throws IOException {
+    return this.dagStateStore.existsDag(dagId.toString());

Review Comment:
   `DagId` seems the better param type for the `DagStateStore` methods than 
`String`.  I suggest we move to that by making default methods in the DSS 
interface that call `.toString()` there, in the interface itself, rather than 
here
   
   then either as part of this PR or leave a TODO to deprecate the `String 
dagId` forms



##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/MostlyInMemoryDagManagementStateStore.java:
##########
@@ -130,44 +122,42 @@ public Set<String> getFailedDagIds() throws IOException {
   }
 
   @Override
-  public synchronized void deleteDagNodeState(String dagId, 
Dag.DagNode<JobExecutionPlan> dagNode) {
+  public synchronized void deleteDagNodeState(DagManager.DagId dagId, 
Dag.DagNode<JobExecutionPlan> dagNode) {
+    String dagIdStr = dagId.toString();
     this.jobToDag.remove(dagNode);
     this.dagNodes.remove(dagNode.getValue().getId());
-    this.dagToDeadline.remove(dagId);
-    this.dagToJobs.get(dagId).remove(dagNode);
-    if (this.dagToJobs.get(dagId).isEmpty()) {
-      this.dagToJobs.remove(dagId);
+    this.dagToDeadline.remove(dagIdStr);
+    this.dagToJobs.get(dagIdStr).remove(dagNode);
+    if (this.dagToJobs.get(dagIdStr).isEmpty()) {
+      this.dagToJobs.remove(dagIdStr);
     }
   }
 
   @Override
-  public synchronized void addDagNodeState(String dagId, 
Dag.DagNode<JobExecutionPlan> dagNode) {
-    Dag<JobExecutionPlan> dag = this.dags.get(dagId);
+  public synchronized void addDagNodeState(DagManager.DagId dagId, 
Dag.DagNode<JobExecutionPlan> dagNode)
+      throws IOException {
+    String dagIdStr = dagId.toString();
+    Dag<JobExecutionPlan> dag = getDag(dagId);
     this.jobToDag.put(dagNode, dag);
     this.dagNodes.put(dagNode.getValue().getId(), dagNode);
-    if (!this.dagToJobs.containsKey(dagId)) {
-      this.dagToJobs.put(dagId, Lists.newLinkedList());
+    if (!this.dagToJobs.containsKey(dagIdStr)) {
+      this.dagToJobs.put(dagIdStr, Lists.newLinkedList());
     }
-    this.dagToJobs.get(dagId).add(dagNode);
-  }
-
-  @Override
-  public Dag<JobExecutionPlan> getDag(DagManager.DagId dagId) {
-    return this.dags.get(dagId.toString());
+    this.dagToJobs.get(dagIdStr).add(dagNode);

Review Comment:
   I didn't see your response to my earlier comment...
   
   given DagId has equals and hash code, could we make it the key type, rather 
than String?



##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/MysqlDagStateStore.java:
##########
@@ -134,6 +134,11 @@ public void cleanUp(String dagId)
     this.totalDagCount.dec();
   }
 
+  @Override
+  public boolean existsDag(String dagId) throws IOException {

Review Comment:
   should this param type be `DagId`, not `String`?



##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManagementStateStore.java:
##########
@@ -32,44 +33,49 @@
  * and allows add/delete and other functions.
  */
 public interface DagManagementStateStore {
-  void addDag(Dag<JobExecutionPlan> dag);
   /**
    * Persist the {@link Dag} to the backing store.
-   * This is not an actual checkpoint but more like a Write-ahead log, where 
uncommitted job will be persisted
-   * and be picked up again when leader transition happens.
+   * It is named this because it is usually called to checkpoint any changes 
in {@link Dag} or in its {@link Dag.DagNode}s.
    * @param dag The dag submitted to {@link DagManager}
    */
   void writeCheckpoint(Dag<JobExecutionPlan> dag) throws IOException;

Review Comment:
   seems like this has been combined w/ `addDag` (which is great!)... but why 
not keep that name rather than this one?  `addDag` seems clearer and also 
aligns better w/ `containsDag` and `getDag`.
   
   relatedly could we rename `cleanUp` to `deleteDag`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to