tvalentyn commented on code in PR #41394:
URL: https://github.com/apache/beam/pull/41394#discussion_r4607437357


##########
sdks/python/apache_beam/runners/worker/bundle_processor.py:
##########
@@ -560,6 +560,9 @@ def commit(self) -> None:
 
 
 class CombiningValueRuntimeState(userstate.CombiningValueRuntimeState):
+  # Probability of compacting persisted bag accumulators on blind-write 
commits.
+  _BLIND_WRITE_COMPACT_PROBABILITY = 0.02

Review Comment:
   how was this selected?



##########
sdks/python/apache_beam/runners/worker/bundle_processor_test.py:
##########
@@ -540,6 +540,138 @@ def test_commit_rejects_failed_set_compaction(self):
       state.commit()
     self.assertEqual(len(futures), 2)
 
+  def test_combining_value_state_per_bundle_accumulator_and_read(self):
+    state_handler = mock.Mock(spec=sdk_worker.CachingStateHandler)
+    state_handler.blocking_get.return_value = [10]
+    state_handler.clear.side_effect = (
+        lambda *args: 
sdk_worker._Future().set(beam_fn_api_pb2.StateResponse()))
+    state_handler.extend.side_effect = (
+        lambda *args: 
sdk_worker._Future().set(beam_fn_api_pb2.StateResponse()))
+
+    state_key = beam_fn_api_pb2.StateKey(
+        bag_user_state=beam_fn_api_pb2.StateKey.BagUserState())
+    bag_state = bundle_processor.SynchronousBagRuntimeState(
+        state_handler, state_key, beam.coders.VarIntCoder())
+    combining_state = bundle_processor.CombiningValueRuntimeState(
+        bag_state, beam.transforms.combiners.CountCombineFn())
+
+    for i in range(1, 101):
+      combining_state.add(1)
+      self.assertEqual(combining_state.read(), 10 + i)
+
+    self.assertEqual(state_handler.blocking_get.call_count, 1)
+    self.assertEqual(state_handler.clear.call_count, 0)
+    self.assertEqual(state_handler.extend.call_count, 0)
+
+    combining_state.commit()
+    self.assertEqual(state_handler.blocking_get.call_count, 1)
+    self.assertEqual(state_handler.clear.call_count, 1)
+    self.assertEqual(state_handler.extend.call_count, 1)
+    self.assertEqual(state_handler.extend.call_args[0][2], [110])
+
+  def test_combining_value_state_blind_write_and_clear(self):
+    state_handler = mock.Mock(spec=sdk_worker.CachingStateHandler)
+    state_handler.blocking_get.return_value = [10, 20]
+    state_handler.clear.side_effect = (
+        lambda *args: 
sdk_worker._Future().set(beam_fn_api_pb2.StateResponse()))
+    state_handler.extend.side_effect = (
+        lambda *args: 
sdk_worker._Future().set(beam_fn_api_pb2.StateResponse()))
+
+    state_key = beam_fn_api_pb2.StateKey(
+        bag_user_state=beam_fn_api_pb2.StateKey.BagUserState())
+    bag_state = bundle_processor.SynchronousBagRuntimeState(
+        state_handler, state_key, beam.coders.VarIntCoder())
+    combining_state = bundle_processor.CombiningValueRuntimeState(
+        bag_state, beam.transforms.combiners.CountCombineFn())
+
+    with mock.patch.object(random, 'random', return_value=0.5):
+      for _ in range(100):
+        combining_state.add(1)
+      combining_state.commit()
+
+    self.assertEqual(state_handler.blocking_get.call_count, 0)
+    self.assertEqual(state_handler.clear.call_count, 0)
+    self.assertEqual(state_handler.extend.call_count, 1)
+    self.assertEqual(state_handler.extend.call_args[0][2], [100])

Review Comment:
   +1 this should help with coverage report above. 



##########
sdks/python/apache_beam/runners/worker/bundle_processor.py:
##########
@@ -560,6 +560,9 @@ def commit(self) -> None:
 
 
 class CombiningValueRuntimeState(userstate.CombiningValueRuntimeState):
+  # Probability of compacting persisted bag accumulators on blind-write 
commits.
+  _BLIND_WRITE_COMPACT_PROBABILITY = 0.02

Review Comment:
   i am also curious whether this can increase memory demands for the runner 
for accumulators that store more than 1 element per key, like TopN. Perhaps we 
can ask an AI tool to run a load test ?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to