This is an automated email from the ASF dual-hosted git repository.
xqhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new d723216eeb3 fixed ML tests (#33234)
d723216eeb3 is described below
commit d723216eeb38698156226a43740f59d4fb751e33
Author: liferoad <[email protected]>
AuthorDate: Wed Nov 27 11:00:52 2024 -0500
fixed ML tests (#33234)
* fixed ML tests
* try some new setups
* created py312-ml tox section
---
sdks/python/apache_beam/ml/transforms/base_test.py | 21 ++++++++++-----------
.../apache_beam/ml/transforms/handlers_test.py | 8 ++++----
sdks/python/setup.py | 20 +++++++++++++++-----
sdks/python/tox.ini | 14 +++++++++++++-
4 files changed, 42 insertions(+), 21 deletions(-)
diff --git a/sdks/python/apache_beam/ml/transforms/base_test.py
b/sdks/python/apache_beam/ml/transforms/base_test.py
index 1a21f6caf7e..3db5a63b954 100644
--- a/sdks/python/apache_beam/ml/transforms/base_test.py
+++ b/sdks/python/apache_beam/ml/transforms/base_test.py
@@ -21,7 +21,6 @@ import secrets
import shutil
import tempfile
import time
-import typing
import unittest
from collections.abc import Sequence
from typing import Any
@@ -140,8 +139,8 @@ class BaseMLTransformTest(unittest.TestCase):
'x': int, 'y': float
},
expected_dtype={
- 'x': typing.Sequence[np.float32],
- 'y': typing.Sequence[np.float32],
+ 'x': Sequence[np.float32],
+ 'y': Sequence[np.float32],
},
),
param(
@@ -153,8 +152,8 @@ class BaseMLTransformTest(unittest.TestCase):
'x': np.int32, 'y': np.float32
},
expected_dtype={
- 'x': typing.Sequence[np.float32],
- 'y': typing.Sequence[np.float32],
+ 'x': Sequence[np.float32],
+ 'y': Sequence[np.float32],
},
),
param(
@@ -165,8 +164,8 @@ class BaseMLTransformTest(unittest.TestCase):
'x': list[int], 'y': list[float]
},
expected_dtype={
- 'x': typing.Sequence[np.float32],
- 'y': typing.Sequence[np.float32],
+ 'x': Sequence[np.float32],
+ 'y': Sequence[np.float32],
},
),
param(
@@ -174,12 +173,12 @@ class BaseMLTransformTest(unittest.TestCase):
'x': [1, 2, 3], 'y': [2.0, 3.0, 4.0]
}],
input_types={
- 'x': typing.Sequence[int],
- 'y': typing.Sequence[float],
+ 'x': Sequence[int],
+ 'y': Sequence[float],
},
expected_dtype={
- 'x': typing.Sequence[np.float32],
- 'y': typing.Sequence[np.float32],
+ 'x': Sequence[np.float32],
+ 'y': Sequence[np.float32],
},
),
])
diff --git a/sdks/python/apache_beam/ml/transforms/handlers_test.py
b/sdks/python/apache_beam/ml/transforms/handlers_test.py
index 4b53026c36a..bb5f9b5f0f7 100644
--- a/sdks/python/apache_beam/ml/transforms/handlers_test.py
+++ b/sdks/python/apache_beam/ml/transforms/handlers_test.py
@@ -20,9 +20,9 @@ import os
import shutil
import sys
import tempfile
-import typing
import unittest
import uuid
+from collections.abc import Sequence
from typing import NamedTuple
from typing import Union
@@ -276,9 +276,9 @@ class TFTProcessHandlerTest(unittest.TestCase):
schema_utils.schema_from_feature_spec(raw_data_feature_spec))
expected_transformed_data_schema = {
- 'x': typing.Sequence[np.float32],
- 'y': typing.Sequence[np.float32],
- 'z': typing.Sequence[bytes]
+ 'x': Sequence[np.float32],
+ 'y': Sequence[np.float32],
+ 'z': Sequence[bytes]
}
actual_transformed_data_schema = (
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 3b45cbf82fc..53c7a532e70 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -490,12 +490,9 @@ if __name__ == '__main__':
'sentence-transformers',
'skl2onnx',
'pillow',
- # Support TF 2.16.0: https://github.com/apache/beam/issues/31294
- # Once TF version is unpinned, also don't restrict Python
version.
- 'tensorflow<2.16.0;python_version<"3.12"',
+ 'tensorflow',
'tensorflow-hub',
- # https://github.com/tensorflow/transform/issues/313
- 'tensorflow-transform;python_version<"3.11"',
+ 'tensorflow-transform',
'tf2onnx',
'torch',
'transformers',
@@ -504,6 +501,19 @@ if __name__ == '__main__':
# https://github.com/apache/beam/issues/31285
# 'xgboost<2.0', # https://github.com/apache/beam/issues/31252
],
+ 'p312_ml_test': [
+ 'datatable',
+ 'embeddings',
+ 'onnxruntime',
+ 'sentence-transformers',
+ 'skl2onnx',
+ 'pillow',
+ 'tensorflow',
+ 'tensorflow-hub',
+ 'tf2onnx',
+ 'torch',
+ 'transformers',
+ ],
'aws': ['boto3>=1.9,<2'],
'azure': [
'azure-storage-blob>=12.3.2,<13',
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 7a242432589..68ac15ced70 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -101,11 +101,23 @@ commands =
python apache_beam/examples/complete/autocomplete_test.py
bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}"
-[testenv:py{39,310,311,312}-ml]
+[testenv:py{39,310,311}-ml]
# Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests.
setenv =
extras = test,gcp,dataframe,ml_test
commands =
+ # Log tensorflow version for debugging
+ /bin/sh -c "pip freeze | grep -E tensorflow"
+ bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}"
+
+[testenv:py312-ml]
+# many packages do not support py3.12
+# Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests.
+setenv =
+extras = test,gcp,dataframe,p312_ml_test
+commands =
+ # Log tensorflow version for debugging
+ /bin/sh -c "pip freeze | grep -E tensorflow"
bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}"
[testenv:py{39,310,311,312}-dask]
