github-actions[bot] commented on code in PR #41394:
URL: https://github.com/apache/beam/pull/41394#discussion_r4607503208


##########
sdks/python/apache_beam/runners/worker/bundle_processor.py:
##########
@@ -568,34 +571,71 @@ def __init__(
     self._combinefn.setup()
     self._underlying_bag_state = underlying_bag_state
     self._finalized = False
-
-  def _read_accumulator(self, rewrite=True):
-    merged_accumulator = self._combinefn.merge_accumulators(
-        self._underlying_bag_state.read())
-    if rewrite:
-      self._underlying_bag_state.clear()
-      self._underlying_bag_state.add(merged_accumulator)
-    return merged_accumulator
+    self._pending_accumulator: Any = None
+    self._has_pending_accumulator = False
+    self._underlying_read = False
+    self._needs_clear = False
+    self._dirty = False
+
+  def _fold_underlying_into_pending(self) -> None:
+    if not self._underlying_read:
+      underlying_accumulators = list(self._underlying_bag_state.read())
+      self._underlying_read = True
+      if underlying_accumulators:
+        self._needs_clear = True
+        if len(underlying_accumulators) > 1:
+          self._dirty = True
+        # Copy the first persisted accumulator so merge_accumulators or
+        # subsequent add_input calls cannot mutate a cached StateCache entry in
+        # place.
+        accumulators = [
+            copy.deepcopy(underlying_accumulators[0]),
+            *underlying_accumulators[1:]
+        ]
+        if self._has_pending_accumulator:
+          accumulators.append(self._pending_accumulator)
+        self._pending_accumulator = self._combinefn.merge_accumulators(
+            accumulators)
+        self._has_pending_accumulator = True
 
   def read(self) -> Iterable[Any]:
-    return self._combinefn.extract_output(self._read_accumulator())
+    self._fold_underlying_into_pending()
+    if not self._has_pending_accumulator:
+      return self._combinefn.extract_output(
+          self._combinefn.create_accumulator())
+    return self._combinefn.extract_output(
+        copy.deepcopy(self._pending_accumulator))
 
   def add(self, value: Any) -> None:
-    # Prefer blind writes, but don't let them grow unboundedly.
-    # This should be tuned to be much lower, but for now exercise
-    # both paths well.
-    if random.random() < 0.5:
-      accumulator = self._read_accumulator(False)
-      self._underlying_bag_state.clear()
-    else:
-      accumulator = self._combinefn.create_accumulator()
-    self._underlying_bag_state.add(
-        self._combinefn.add_input(accumulator, value))
+    if not self._has_pending_accumulator:
+      self._pending_accumulator = self._combinefn.create_accumulator()
+      self._has_pending_accumulator = True
+    self._pending_accumulator = self._combinefn.add_input(
+        self._pending_accumulator, value)
+    self._dirty = True
 
   def clear(self) -> None:
     self._underlying_bag_state.clear()
+    self._pending_accumulator = None
+    self._has_pending_accumulator = False
+    self._underlying_read = True
+    self._needs_clear = True
+    self._dirty = True
 
-  def commit(self):
+  def commit(self) -> None:
+    if self._dirty:
+      if self._has_pending_accumulator and not self._underlying_read:
+        if random.random() < self._BLIND_WRITE_COMPACT_PROBABILITY:
+          self._fold_underlying_into_pending()
+      if self._needs_clear:
+        self._underlying_bag_state.clear()
+      if self._has_pending_accumulator:
+        self._underlying_bag_state.add(self._pending_accumulator)

Review Comment:
   **🟢 Low**: Consider calling 
`self._combinefn.compact(self._pending_accumulator)` before adding the 
accumulator to `_underlying_bag_state`. Certain `CombineFn` implementations 
rely on `compact()` before serialization or cross-bundle caching (for example, 
`CallableWrapperCombineFn` reduces its buffered input list, and `TopCombineFn` 
unwraps `ComparableValue` instances to avoid pickling comparator functions).
   
   ```suggestion
           
self._underlying_bag_state.add(self._combinefn.compact(self._pending_accumulator))
   ```



##########
sdks/python/apache_beam/runners/worker/bundle_processor_test.py:
##########
@@ -540,6 +540,138 @@ def test_commit_rejects_failed_set_compaction(self):
       state.commit()
     self.assertEqual(len(futures), 2)
 
+  def test_combining_value_state_per_bundle_accumulator_and_read(self):
+    state_handler = mock.Mock(spec=sdk_worker.CachingStateHandler)
+    state_handler.blocking_get.return_value = [10]
+    state_handler.clear.side_effect = (
+        lambda *args: 
sdk_worker._Future().set(beam_fn_api_pb2.StateResponse()))
+    state_handler.extend.side_effect = (
+        lambda *args: 
sdk_worker._Future().set(beam_fn_api_pb2.StateResponse()))
+
+    state_key = beam_fn_api_pb2.StateKey(
+        bag_user_state=beam_fn_api_pb2.StateKey.BagUserState())
+    bag_state = bundle_processor.SynchronousBagRuntimeState(
+        state_handler, state_key, beam.coders.VarIntCoder())
+    combining_state = bundle_processor.CombiningValueRuntimeState(
+        bag_state, beam.transforms.combiners.CountCombineFn())
+
+    for i in range(1, 101):
+      combining_state.add(1)
+      self.assertEqual(combining_state.read(), 10 + i)
+
+    self.assertEqual(state_handler.blocking_get.call_count, 1)
+    self.assertEqual(state_handler.clear.call_count, 0)
+    self.assertEqual(state_handler.extend.call_count, 0)
+
+    combining_state.commit()
+    self.assertEqual(state_handler.blocking_get.call_count, 1)
+    self.assertEqual(state_handler.clear.call_count, 1)
+    self.assertEqual(state_handler.extend.call_count, 1)
+    self.assertEqual(state_handler.extend.call_args[0][2], [110])
+
+  def test_combining_value_state_blind_write_and_clear(self):
+    state_handler = mock.Mock(spec=sdk_worker.CachingStateHandler)
+    state_handler.blocking_get.return_value = [10, 20]
+    state_handler.clear.side_effect = (
+        lambda *args: 
sdk_worker._Future().set(beam_fn_api_pb2.StateResponse()))
+    state_handler.extend.side_effect = (
+        lambda *args: 
sdk_worker._Future().set(beam_fn_api_pb2.StateResponse()))
+
+    state_key = beam_fn_api_pb2.StateKey(
+        bag_user_state=beam_fn_api_pb2.StateKey.BagUserState())
+    bag_state = bundle_processor.SynchronousBagRuntimeState(
+        state_handler, state_key, beam.coders.VarIntCoder())
+    combining_state = bundle_processor.CombiningValueRuntimeState(
+        bag_state, beam.transforms.combiners.CountCombineFn())
+
+    with mock.patch.object(random, 'random', return_value=0.5):
+      for _ in range(100):
+        combining_state.add(1)
+      combining_state.commit()
+
+    self.assertEqual(state_handler.blocking_get.call_count, 0)
+    self.assertEqual(state_handler.clear.call_count, 0)
+    self.assertEqual(state_handler.extend.call_count, 1)
+    self.assertEqual(state_handler.extend.call_args[0][2], [100])

Review Comment:
   **🟢 Low**: `test_combining_value_state_blind_write_and_clear` patches 
`random.random` to return `0.5` (`>= _BLIND_WRITE_COMPACT_PROBABILITY`), 
leaving the probabilistic compaction branch in `commit()` (`random.random() < 
self._BLIND_WRITE_COMPACT_PROBABILITY` on lines 628-629 of 
`bundle_processor.py`) untested. Consider adding an assertion path where 
`random.random` returns `< 0.02` (or relies on the `setUp` default of `0`) to 
verify that blind-write compaction folds the underlying state `[10, 20]`, 
clears the bag, and writes the merged accumulator `[130]`.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to