uranusjr commented on code in PR #68939:
URL: https://github.com/apache/airflow/pull/68939#discussion_r3490513052
##########
airflow-e2e-tests/tests/airflow_e2e_tests/constants.py:
##########
@@ -58,6 +58,11 @@
JAVA_COMPOSE_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" /
"java.yml"
JAVA_DOCKERFILE_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" /
"Dockerfile.java"
+# Scala Spark example is a separate Java-SDK bundle, served by its own
+# coordinator/queue but exercised within the same java_sdk E2E mode.
Review Comment:
Also too verbose
##########
airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py:
##########
@@ -347,17 +420,35 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir):
check=True,
)
- # Coordinator registry: maps the logical name "java-jdk" to
JavaCoordinator.
- # Queue mapping: routes tasks on the "java" Celery queue to "java-jdk".
+ # Coordinator registry: two JavaCoordinators on the same worker image,
+ # serving different bundles on different queues.
+ # java-jdk -> the lean Java example bundle (queue "java")
+ # scala-jdk -> the Scala Spark bundle (queue "scala"), which needs
Spark's
+ # Java 17 module openings, a small driver heap, and a longer
+ # startup timeout for its large dependency classpath.
coordinator_config = json.dumps(
{
"java-jdk": {
"classpath": "airflow.sdk.coordinators.java.JavaCoordinator",
- "kwargs": {"jars_root": ["/opt/airflow/jars"]},
- }
+ "kwargs": {"jars_root": ["/opt/airflow/java-jars"]},
+ },
+ "scala-jdk": {
+ "classpath": "airflow.sdk.coordinators.java.JavaCoordinator",
+ "kwargs": {
+ "jars_root": ["/opt/airflow/scala-jars"],
+ # Pin the entry point so Spark's large dependency classpath
+ # cannot make Main-Class discovery ambiguous.
+ "main_class":
"org.apache.airflow.example.ScalaSparkBundleBuilder",
+ # Small driver heap plus Spark's full Java 17 module
openings.
+ "jvm_args": ["-Xmx512m", *_SPARK_JAVA_MODULE_OPTIONS],
+ # Booting a JVM over the large Spark classpath takes longer
+ # than the 10 s default before it connects back.
+ "task_startup_timeout": 60.0,
Review Comment:
Comments here are too verbose, probably not needed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
