[ 
https://issues.apache.org/jira/browse/GOBBLIN-2002?focusedWorklogId=905016&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-905016
 ]

ASF GitHub Bot logged work on GOBBLIN-2002:
-------------------------------------------

                Author: ASF GitHub Bot
            Created on: 15/Feb/24 07:20
            Start Date: 15/Feb/24 07:20
    Worklog Time Spent: 10m 
      Work Description: phet commented on code in PR #3878:
URL: https://github.com/apache/gobblin/pull/3878#discussion_r1490500328


##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagStateStore.java:
##########
@@ -70,4 +71,8 @@ public interface DagStateStore {
    * Return a list of all dag IDs contained in the dag state store.
    */
   Set<String> getDagIds() throws IOException;
+
+  default boolean existsDag(String dagId) throws IOException {
+    throw new UnsupportedEncodingException("containsDag not implemented");

Review Comment:
   NBD, but I believe you want `java.lang.UnsupportedOperationException`



##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/MostlyInMemoryDagManagementStateStore.java:
##########
@@ -130,44 +122,42 @@ public Set<String> getFailedDagIds() throws IOException {
   }
 
   @Override
-  public synchronized void deleteDagNodeState(String dagId, 
Dag.DagNode<JobExecutionPlan> dagNode) {
+  public synchronized void deleteDagNodeState(DagManager.DagId dagId, 
Dag.DagNode<JobExecutionPlan> dagNode) {
+    String dagIdStr = dagId.toString();
     this.jobToDag.remove(dagNode);
     this.dagNodes.remove(dagNode.getValue().getId());
-    this.dagToDeadline.remove(dagId);
-    this.dagToJobs.get(dagId).remove(dagNode);
-    if (this.dagToJobs.get(dagId).isEmpty()) {
-      this.dagToJobs.remove(dagId);
+    this.dagToDeadline.remove(dagIdStr);
+    this.dagToJobs.get(dagIdStr).remove(dagNode);
+    if (this.dagToJobs.get(dagIdStr).isEmpty()) {
+      this.dagToJobs.remove(dagIdStr);
     }
   }
 
   @Override
-  public synchronized void addDagNodeState(String dagId, 
Dag.DagNode<JobExecutionPlan> dagNode) {
-    Dag<JobExecutionPlan> dag = this.dags.get(dagId);
+  public synchronized void addDagNodeState(DagManager.DagId dagId, 
Dag.DagNode<JobExecutionPlan> dagNode)
+      throws IOException {
+    String dagIdStr = dagId.toString();
+    Dag<JobExecutionPlan> dag = getDag(dagId);
     this.jobToDag.put(dagNode, dag);
     this.dagNodes.put(dagNode.getValue().getId(), dagNode);
-    if (!this.dagToJobs.containsKey(dagId)) {
-      this.dagToJobs.put(dagId, Lists.newLinkedList());
+    if (!this.dagToJobs.containsKey(dagIdStr)) {
+      this.dagToJobs.put(dagIdStr, Lists.newLinkedList());
     }
-    this.dagToJobs.get(dagId).add(dagNode);
-  }
-
-  @Override
-  public Dag<JobExecutionPlan> getDag(DagManager.DagId dagId) {
-    return this.dags.get(dagId.toString());
+    this.dagToJobs.get(dagIdStr).add(dagNode);
   }
 
   @Override
-  public void addDag(Dag<JobExecutionPlan> dag) {
-    this.dags.put(DagManagerUtils.generateDagId(dag).toString(), dag);
+  public Dag<JobExecutionPlan> getDag(DagManager.DagId dagId) throws 
IOException {
+    return this.dagStateStore.getDag(dagId.toString());
   }
 
   @Override
-  public boolean containsDag(String dagId) {
-    return this.dags.containsKey(dagId);
+  public boolean containsDag(DagManager.DagId dagId) throws IOException {
+    return this.dagStateStore.existsDag(dagId.toString());

Review Comment:
   `DagId` seems the better param type for the `DagStateStore` methods than 
`String`.  I suggest we move to that by making default methods in the DSS 
interface that call `.toString()` there, in the interface itself, rather than 
here
   
   then either as part of this PR or leave a TODO to deprecate the `String 
dagId` forms



##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/MostlyInMemoryDagManagementStateStore.java:
##########
@@ -130,44 +122,42 @@ public Set<String> getFailedDagIds() throws IOException {
   }
 
   @Override
-  public synchronized void deleteDagNodeState(String dagId, 
Dag.DagNode<JobExecutionPlan> dagNode) {
+  public synchronized void deleteDagNodeState(DagManager.DagId dagId, 
Dag.DagNode<JobExecutionPlan> dagNode) {
+    String dagIdStr = dagId.toString();
     this.jobToDag.remove(dagNode);
     this.dagNodes.remove(dagNode.getValue().getId());
-    this.dagToDeadline.remove(dagId);
-    this.dagToJobs.get(dagId).remove(dagNode);
-    if (this.dagToJobs.get(dagId).isEmpty()) {
-      this.dagToJobs.remove(dagId);
+    this.dagToDeadline.remove(dagIdStr);
+    this.dagToJobs.get(dagIdStr).remove(dagNode);
+    if (this.dagToJobs.get(dagIdStr).isEmpty()) {
+      this.dagToJobs.remove(dagIdStr);
     }
   }
 
   @Override
-  public synchronized void addDagNodeState(String dagId, 
Dag.DagNode<JobExecutionPlan> dagNode) {
-    Dag<JobExecutionPlan> dag = this.dags.get(dagId);
+  public synchronized void addDagNodeState(DagManager.DagId dagId, 
Dag.DagNode<JobExecutionPlan> dagNode)
+      throws IOException {
+    String dagIdStr = dagId.toString();
+    Dag<JobExecutionPlan> dag = getDag(dagId);
     this.jobToDag.put(dagNode, dag);
     this.dagNodes.put(dagNode.getValue().getId(), dagNode);
-    if (!this.dagToJobs.containsKey(dagId)) {
-      this.dagToJobs.put(dagId, Lists.newLinkedList());
+    if (!this.dagToJobs.containsKey(dagIdStr)) {
+      this.dagToJobs.put(dagIdStr, Lists.newLinkedList());
     }
-    this.dagToJobs.get(dagId).add(dagNode);
-  }
-
-  @Override
-  public Dag<JobExecutionPlan> getDag(DagManager.DagId dagId) {
-    return this.dags.get(dagId.toString());
+    this.dagToJobs.get(dagIdStr).add(dagNode);

Review Comment:
   I didn't see your response to my earlier comment...
   
   given DagId has equals and hash code, could we make it the key type, rather 
than String?



##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/MysqlDagStateStore.java:
##########
@@ -134,6 +134,11 @@ public void cleanUp(String dagId)
     this.totalDagCount.dec();
   }
 
+  @Override
+  public boolean existsDag(String dagId) throws IOException {

Review Comment:
   should this param type be `DagId`, not `String`?



##########
gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManagementStateStore.java:
##########
@@ -32,44 +33,49 @@
  * and allows add/delete and other functions.
  */
 public interface DagManagementStateStore {
-  void addDag(Dag<JobExecutionPlan> dag);
   /**
    * Persist the {@link Dag} to the backing store.
-   * This is not an actual checkpoint but more like a Write-ahead log, where 
uncommitted job will be persisted
-   * and be picked up again when leader transition happens.
+   * It is named this because it is usually called to checkpoint any changes 
in {@link Dag} or in its {@link Dag.DagNode}s.
    * @param dag The dag submitted to {@link DagManager}
    */
   void writeCheckpoint(Dag<JobExecutionPlan> dag) throws IOException;

Review Comment:
   seems like this has been combined w/ `addDag` (which is great!)... but why 
not keep that name rather than this one?  `addDag` seems clearer and also 
aligns better w/ `containsDag` and `getDag`.
   
   relatedly could we rename `cleanUp` to `deleteDag`?





Issue Time Tracking
-------------------

    Worklog Id:     (was: 905016)
    Time Spent: 4h  (was: 3h 50m)

> create MostlyInMemoryDagManagementStateStore to merge UserQuotaManager, 
> DagStateStore and in-memory dag maps used in DagManager
> -------------------------------------------------------------------------------------------------------------------------------
>
>                 Key: GOBBLIN-2002
>                 URL: https://issues.apache.org/jira/browse/GOBBLIN-2002
>             Project: Apache Gobblin
>          Issue Type: Improvement
>            Reporter: Arjun Singh Bora
>            Priority: Major
>          Time Spent: 4h
>  Remaining Estimate: 0h
>
> this will help in refactoring DagManager



--
This message was sent by Atlassian Jira
(v8.20.10#820010)

Reply via email to