This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new c0ab7e5ac30 ZetaSQL test Java version fixes (#33213)
c0ab7e5ac30 is described below
commit c0ab7e5ac303575dbc7a79b92c87282183ef6f69
Author: Yi Hu <[email protected]>
AuthorDate: Mon Nov 25 17:04:23 2024 -0500
ZetaSQL test Java version fixes (#33213)
* Align SDK container version with pipeline submission env
* Disable ZetaSQL test on Java8
---
...m_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.json | 1 +
.github/trigger_files/beam_PostCommit_XVR_Samza.json | 1 +
CHANGES.md | 1 +
runners/google-cloud-dataflow-java/build.gradle | 4 +++-
sdks/python/apache_beam/transforms/sql_test.py | 20 ++++++++++++++++++++
5 files changed, 26 insertions(+), 1 deletion(-)
diff --git
a/.github/trigger_files/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.json
b/.github/trigger_files/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.json
new file mode 100644
index 00000000000..9e26dfeeb6e
--- /dev/null
+++ b/.github/trigger_files/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.json
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/.github/trigger_files/beam_PostCommit_XVR_Samza.json
b/.github/trigger_files/beam_PostCommit_XVR_Samza.json
new file mode 100644
index 00000000000..9e26dfeeb6e
--- /dev/null
+++ b/.github/trigger_files/beam_PostCommit_XVR_Samza.json
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/CHANGES.md b/CHANGES.md
index 7a5e8942565..654512c3a4e 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -70,6 +70,7 @@
## Breaking Changes
+* Upgraded ZetaSQL to 2024.11.1
([#32902](https://github.com/apache/beam/pull/32902)). Java11+ is now needed if
Beam's ZetaSQL component is used.
* X behavior was changed ([#X](https://github.com/apache/beam/issues/X)).
## Deprecations
diff --git a/runners/google-cloud-dataflow-java/build.gradle
b/runners/google-cloud-dataflow-java/build.gradle
index 3a337684bf1..811a3c15f83 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -16,6 +16,8 @@
* limitations under the License.
*/
+import static org.apache.beam.gradle.BeamModulePlugin.getSupportedJavaVersion
+
import groovy.json.JsonOutput
plugins { id 'org.apache.beam.module' }
@@ -341,7 +343,7 @@ tasks.register('buildAndPushDistrolessContainerImage',
Task.class) {
// task directly ('dependsOn buildAndPushDockerJavaContainer'). This ensures
the correct
// task ordering such that the registry doesn't get cleaned up prior to task
completion.
def buildAndPushDockerJavaContainer =
tasks.register("buildAndPushDockerJavaContainer") {
- def javaVer = "java8"
+ def javaVer = getSupportedJavaVersion()
if(project.hasProperty('testJavaVersion')) {
javaVer = "java${project.getProperty('testJavaVersion')}"
}
diff --git a/sdks/python/apache_beam/transforms/sql_test.py
b/sdks/python/apache_beam/transforms/sql_test.py
index 854aec078ce..a7da253c461 100644
--- a/sdks/python/apache_beam/transforms/sql_test.py
+++ b/sdks/python/apache_beam/transforms/sql_test.py
@@ -20,6 +20,7 @@
# pytype: skip-file
import logging
+import subprocess
import typing
import unittest
@@ -69,6 +70,22 @@ class SqlTransformTest(unittest.TestCase):
"""
_multiprocess_can_split_ = True
+ @staticmethod
+ def _disable_zetasql_test():
+ # disable if run on Java8 which is no longer supported by ZetaSQL
+ try:
+ result = subprocess.run(['java', '-version'],
+ check=True,
+ capture_output=True,
+ text=True)
+ version_line = result.stderr.splitlines()[0]
+ version = version_line.split()[2].strip('\"')
+ if version.startswith("1."):
+ return True
+ return False
+ except: # pylint: disable=bare-except
+ return False
+
def test_generate_data(self):
with TestPipeline() as p:
out = p | SqlTransform(
@@ -150,6 +167,9 @@ class SqlTransformTest(unittest.TestCase):
assert_that(out, equal_to([(1, 1), (4, 1), (100, 2)]))
def test_zetasql_generate_data(self):
+ if self._disable_zetasql_test():
+ raise unittest.SkipTest("ZetaSQL tests need Java11+")
+
with TestPipeline() as p:
out = p | SqlTransform(
"""SELECT
