gaogaotiantian commented on code in PR #57093:
URL: https://github.com/apache/spark/pull/57093#discussion_r3540693567
##########
pyproject.toml:
##########
@@ -68,3 +68,137 @@ line-length = 100
target-version = ['py39']
include = '\.pyi?$'
extend-exclude = 'cloudpickle|error_classes.py'
+
+[dependency-groups]
+_py4j = ["py4j>=0.10.9.9"]
+
+_numpy = ["numpy>=1.23.2"]
+_pandas = ["pandas>=2.2.0,<3.0.0"]
+_protobuf = ["protobuf==6.33.5"]
+_pyarrow = ["pyarrow>=18.0.0"]
+
+_pyspark_devtool = [
+ # Optional tools that are used by pyspark directly.
+ "memory-profiler>=0.61.0",
+ "flameprof==0.4",
+ "psutil",
+ "pystack>=1.6.0; sys_platform=='linux'",
+]
+
+connect = [
+ {include-group = "_py4j"},
+ {include-group = "_pandas"},
+ {include-group = "_protobuf"},
+ {include-group = "_pyarrow"},
+ "grpcio>=1.76.0",
+ "grpcio-status>=1.76.0",
+ "googleapis-common-protos>=1.71.0",
+ "zstandard>=0.25.0",
+ "graphviz>=0.20",
+]
+
+sql = [
+ {include-group = "_py4j"},
+ {include-group = "_pandas"},
+ {include-group = "_pyarrow"},
+]
+
+pandas_on_spark = [
+ {include-group = "_py4j"},
+ {include-group = "_pandas"},
+ {include-group = "_pyarrow"},
+]
+
+pandas_on_spark_extra = [
+ "plotly<6.0.0",
+ "matplotlib",
+ "mlflow>=2.3.1",
+]
+
+ml = [
+ {include-group = "_py4j"},
+ {include-group = "_numpy"},
+]
+
+mllib = [
+ {include-group = "_py4j"},
+ {include-group = "_numpy"},
+]
+
+ml_extra = [
+ "scipy>1.8.0",
+ "scikit-learn",
+ "torch",
+ "torchvision",
+ "torcheval",
+ "deepspeed; sys_platform != 'darwin'",
+]
+
+pipelines = [
+ {include-group = "connect"},
+ "pyyaml>=3.11",
+]
+
+_lint = [
+ "ruff==0.14.8",
+ "mypy==1.19.1",
+ "pytest-mypy-plugins==3.2.0",
+ "mypy-protobuf==3.3.0",
+ "googleapis-common-protos-stubs==2.2.0",
+ "grpc-stubs==1.24.11",
+ "pandas-stubs>=2.2.0",
+ "scipy-stubs",
+ "types-PyYAML"
+]
+
+_test = [
+ # Dependency for running tests
+ "unittest-xml-reporting",
+ # 3rd party libraries that unittests use.
+ "openpyxl",
+ "tabulate",
+ "testcontainers[kafka]>=3.7.0",
+ "kafka-python-ng>=2.0.2",
+]
+
+_docs = [
+ "sphinx==8.2.3",
+ "sphinx-plotly-directive",
+ "sphinx-copybutton",
+ "docutils",
+ "markupsafe",
+ "pydata_sphinx_theme>=0.13",
+ "ipython",
+ "nbsphinx",
+ "numpydoc",
+ "jinja2",
+]
+
+_devtools = [
+ # Devtools that are used by pyspark developers.
+ "debugpy",
+ "viztracer",
+ "jira>=3.5.2",
+ "PyGithub",
+ "asv",
+]
+
+dev = [
Review Comment:
It makes sense to keep this PR only about structure change so I can keep
everything in `dev`.
One thing that's worth discussing is how many packages we want users to
install when they try to work on pyspark. The reason I tried to rule out
`torch` is because `torch` is very large. The wheel is 500M for Linux and it
takes about 1.7G disk after unzip.
`torch` itself is only used for some features of `ml` and in a few tests.
It's also listed as optional for pyspark installation
(https://spark.apache.org/docs/latest/api/python/getting_started/install.html).
I think it's a bit of a waste for users who don't deal with `ml` module to
install `torch`.
However, keeping it exactly as before does not make it worse so we can
definitely do that. What's your opinion on this?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
