This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 3697bcdd51 GH-38060: [Python][CI] Upgrade Spark versions (#38082)
3697bcdd51 is described below
commit 3697bcdd514fc5a0472dabd3928f6b4731554cda
Author: Dane Pitkin <[email protected]>
AuthorDate: Fri Oct 6 16:54:52 2023 -0400
GH-38060: [Python][CI] Upgrade Spark versions (#38082)
### Rationale for this change
Spark released 3.5.0 and dropped support for Java 8 and 11 on main.
### What changes are included in this PR?
* Upgrade Spark integration tests from v3.4.1 -> 3.5.0
* Use JDK 17 for Spark latest
### Are these changes tested?
Will test in CI
### Are there any user-facing changes?
No
* Closes: #38060
Authored-by: Dane Pitkin <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
dev/tasks/tasks.yml | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 798932db23..bb6cdb2125 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -1611,9 +1611,9 @@ tasks:
image: conda-python-hdfs
{% endfor %}
-{% for python_version, spark_version, test_pyarrow_only, numpy_version in
[("3.8", "v3.4.1", "false", "latest"),
-
("3.10", "v3.4.1", "false", "1.23"),
-
("3.11", "master", "false", "latest")] %}
+{% for python_version, spark_version, test_pyarrow_only, numpy_version,
jdk_version in [("3.8", "v3.5.0", "false", "latest", "8"),
+
("3.10", "v3.5.0", "false", "1.23", "8"),
+
("3.11", "master", "false", "latest", "17")] %}
test-conda-python-{{ python_version }}-spark-{{ spark_version }}:
ci: github
template: docker-tests/github.linux.yml
@@ -1623,6 +1623,7 @@ tasks:
SPARK: "{{ spark_version }}"
TEST_PYARROW_ONLY: "{{ test_pyarrow_only }}"
NUMPY: "{{ numpy_version }}"
+ JDK: "{{ jdk_version }}"
# use the branch-3.0 of spark, so prevent reusing any layers
flags: --no-leaf-cache
image: conda-python-spark
