claudevdm commented on code in PR #41361:
URL: https://github.com/apache/beam/pull/41361#discussion_r4586223052
##########
sdks/python/apache_beam/ml/inference/gemini_inference.py:
##########
@@ -226,8 +227,10 @@ def __init__(
self._env_vars = kwargs.get('env_vars', {})
if min_batch_size is not None:
self._batching_kwargs["min_batch_size"] = min_batch_size
- if max_batch_size is not None:
- self._batching_kwargs["max_batch_size"] = max_batch_size
+ if max_batch_size is None:
+ max_batch_size = max(
Review Comment:
What happens when a handler is built with `element_size_fn` and no
`max_batch_weight`? As far as I can tell BatchElements then counts
`max_batch_size` in `element_size_fn` units, so would this default mean 32
characters (or tokens) rather than 32 prompts? With `element_size_fn=len` and
3,000 prompts of 43 to 123 characters I get batches of up to 121 prompts on
master and 3,000 batches of one on this branch (3.4 s against 34.5 s with a 10
ms fake call). It needs prompts of different lengths to show: with identical
lengths master also sends one per batch.
GeminiModelHandler('m', fn, api_key='k',
element_size_fn=len).batch_elements_kwargs()
# master: {'element_size_fn': len}
# branch: {'max_batch_size': 32, 'element_size_fn': len}
Would it work to skip the default in that case, as below? The "Defaults to
32" docstring would need the same exception. With this and the tests in the
next comment, all 66 tests in both files pass.
```suggestion
# With element_size_fn and no max_batch_weight, BatchElements counts
# max_batch_size in element_size_fn units (e.g. characters), not prompts,
# so the default number of prompts does not apply.
sized_by_element_size_fn = (
element_size_fn is not None and max_batch_weight is None)
if max_batch_size is None and not sized_by_element_size_fn:
max_batch_size = max(
min_batch_size or 0, _MAX_CONCURRENT_REQUESTS_PER_BATCH)
if max_batch_size is not None:
self._batching_kwargs["max_batch_size"] = max_batch_size
```
##########
sdks/python/apache_beam/ml/inference/gemini_inference_test.py:
##########
@@ -151,6 +151,24 @@ def test_missing_all_params(self):
request_fn=generate_from_string,
)
+ def test_default_max_batch_size(self):
+ handler = GeminiModelHandler(
+ model_name="gemini-model-123",
+ request_fn=generate_from_string,
+ api_key="123456789")
+ self.assertEqual(handler.batch_elements_kwargs(), {'max_batch_size': 32})
+
+ def test_default_max_batch_size_is_not_below_min_batch_size(self):
+ handler = GeminiModelHandler(
+ model_name="gemini-model-123",
+ request_fn=generate_from_string,
+ api_key="123456789",
+ min_batch_size=64)
+ self.assertEqual(
+ handler.batch_elements_kwargs(), {
+ 'min_batch_size': 64, 'max_batch_size': 64
+ })
+
Review Comment:
Would tests like these be worth adding next to yours? The first one fails on
this branch and passes with the change above; the second checks that the
default still applies when `max_batch_weight` is set.
```suggestion
def test_no_default_max_batch_size_when_size_is_element_size_fn(self):
# Without max_batch_weight, max_batch_size is counted in element_size_fn
# units, so a default of 32 would mean 32 characters here.
handler = GeminiModelHandler(
model_name="gemini-model-123",
request_fn=generate_from_string,
api_key="123456789",
element_size_fn=len)
self.assertEqual(handler.batch_elements_kwargs(), {'element_size_fn':
len})
def test_default_max_batch_size_with_max_batch_weight(self):
handler = GeminiModelHandler(
model_name="gemini-model-123",
request_fn=generate_from_string,
api_key="123456789",
element_size_fn=len,
max_batch_weight=1000)
self.assertEqual(
handler.batch_elements_kwargs(), {
'max_batch_size': 32,
'max_batch_weight': 1000,
'element_size_fn': len
})
```
##########
sdks/python/apache_beam/ml/inference/openai_inference_test.py:
##########
@@ -479,6 +479,31 @@ def test_batch_elements_kwargs(self):
self.assertEqual(kwargs['max_batch_duration_secs'], 5)
self.assertEqual(kwargs['max_batch_weight'], 100)
+ def test_default_max_batch_size(self):
+ handler = OpenAIModelHandler(
+ model_name=_TEST_MODEL,
+ request_fn=chat_completion_from_string,
+ api_key='fake')
+ self.assertEqual(handler.batch_elements_kwargs(), {'max_batch_size': 32})
+
+ def test_default_max_batch_size_for_embeddings(self):
+ handler = OpenAIModelHandler(
+ model_name=_TEST_EMBEDDING_MODEL,
+ request_fn=embedding_from_string,
+ api_key='fake')
+ self.assertEqual(handler.batch_elements_kwargs(), {'max_batch_size': 2048})
+
+ def test_default_max_batch_size_is_not_below_min_batch_size(self):
+ handler = OpenAIModelHandler(
+ model_name=_TEST_MODEL,
+ request_fn=chat_completion_from_string,
+ api_key='fake',
+ min_batch_size=64)
+ self.assertEqual(
+ handler.batch_elements_kwargs(), {
+ 'min_batch_size': 64, 'max_batch_size': 64
+ })
+
Review Comment:
Would the same two tests make sense here, for chat and embeddings? The first
fails on this branch and passes with the change above.
```suggestion
def test_no_default_max_batch_size_when_size_is_element_size_fn(self):
# Without max_batch_weight, max_batch_size is counted in element_size_fn
# units, so a default of 32 or 2048 would mean that many characters here.
for request_fn in (chat_completion_from_string, embedding_from_string):
handler = OpenAIModelHandler(
model_name=_TEST_MODEL,
request_fn=request_fn,
api_key='fake',
element_size_fn=len)
self.assertEqual(
handler.batch_elements_kwargs(), {'element_size_fn': len})
def test_default_max_batch_size_with_max_batch_weight(self):
handler = OpenAIModelHandler(
model_name=_TEST_EMBEDDING_MODEL,
request_fn=embedding_from_string,
api_key='fake',
element_size_fn=len,
max_batch_weight=1000)
self.assertEqual(
handler.batch_elements_kwargs(),
{
'max_batch_size': 2048,
'max_batch_weight': 1000,
'element_size_fn': len
})
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]