This is an automated email from the ASF dual-hosted git repository.
adutra pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 6df5117d Move QuarkusSparkIT to a separate module (#1109)
6df5117d is described below
commit 6df5117d6f232ab0b24dd14cb0ee885e33375225
Author: Alexandre Dutra <[email protected]>
AuthorDate: Thu Mar 6 19:26:47 2025 +0100
Move QuarkusSparkIT to a separate module (#1109)
---
gradle/projects.main.properties | 1 +
quarkus/service/build.gradle.kts | 69 ++----------
.../quarkus/catalog/BasePolarisCatalogTest.java | 16 +--
.../task/ManifestFileCleanupTaskHandlerTest.java | 16 +--
quarkus/spark-tests/build.gradle.kts | 119 +++++++++++++++++++++
.../service/quarkus/spark}/it/QuarkusSparkIT.java | 5 +-
...che.polaris.service.it.ext.PolarisServerManager | 20 ++++
7 files changed, 164 insertions(+), 82 deletions(-)
diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties
index 468cbdf7..4f13a8cd 100644
--- a/gradle/projects.main.properties
+++ b/gradle/projects.main.properties
@@ -26,6 +26,7 @@ polaris-service-common=service/common
polaris-quarkus-defaults=quarkus/defaults
polaris-quarkus-service=quarkus/service
polaris-quarkus-server=quarkus/server
+polaris-quarkus-spark-tests=quarkus/spark-tests
polaris-quarkus-admin=quarkus/admin
polaris-eclipselink=extension/persistence/eclipselink
polaris-jpa-model=extension/persistence/jpa-model
diff --git a/quarkus/service/build.gradle.kts b/quarkus/service/build.gradle.kts
index 7ab5ec87..dfa2d97b 100644
--- a/quarkus/service/build.gradle.kts
+++ b/quarkus/service/build.gradle.kts
@@ -87,7 +87,12 @@ dependencies {
implementation(libs.jakarta.servlet.api)
- testFixturesApi(project(":polaris-tests"))
+ testFixturesApi(project(":polaris-tests")) {
+ // exclude all spark dependencies
+ exclude(group = "org.apache.iceberg", module = "iceberg-spark-3.5_2.12")
+ exclude(group = "org.apache.iceberg", module =
"iceberg-spark-extensions-3.5_2.12")
+ exclude(group = "org.apache.spark", module = "spark-sql_2.12")
+ }
testImplementation(project(":polaris-api-management-model"))
testImplementation(testFixtures(project(":polaris-service-common")))
@@ -95,16 +100,6 @@ dependencies {
testImplementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests")
testImplementation("org.apache.iceberg:iceberg-core:${libs.versions.iceberg.get()}:tests")
- testImplementation("org.apache.iceberg:iceberg-spark-3.5_2.12")
- testImplementation("org.apache.iceberg:iceberg-spark-extensions-3.5_2.12")
- testImplementation(libs.spark35.sql.scala212) {
- // exclude log4j dependencies
- exclude("org.apache.logging.log4j", "log4j-slf4j2-impl")
- exclude("org.apache.logging.log4j", "log4j-api")
- exclude("org.apache.logging.log4j", "log4j-1.2-api")
- exclude("org.slf4j", "jul-to-slf4j")
- }
-
testImplementation("software.amazon.awssdk:glue")
testImplementation("software.amazon.awssdk:kms")
testImplementation("software.amazon.awssdk:dynamodb")
@@ -116,27 +111,8 @@ dependencies {
testImplementation("io.quarkus:quarkus-rest-client-jackson")
testImplementation("io.rest-assured:rest-assured")
- testImplementation(platform(libs.testcontainers.bom))
- testImplementation("org.testcontainers:testcontainers")
- testImplementation(libs.s3mock.testcontainers)
-
+ testImplementation(libs.threeten.extra)
testImplementation(libs.hawkular.agent.prometheus.scraper)
-
- intTestImplementation(project(":polaris-api-management-model"))
-
-
intTestImplementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}")
-
intTestImplementation("org.apache.iceberg:iceberg-core:${libs.versions.iceberg.get()}")
-
intTestImplementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests")
-
intTestImplementation("org.apache.iceberg:iceberg-core:${libs.versions.iceberg.get()}:tests")
-
- intTestImplementation(platform(libs.quarkus.bom))
- intTestImplementation("io.quarkus:quarkus-junit5")
-
- // required for QuarkusSparkIT
- intTestImplementation(enforcedPlatform(libs.scala212.lang.library))
- intTestImplementation(enforcedPlatform(libs.scala212.lang.reflect))
- intTestImplementation(libs.javax.servlet.api)
- intTestImplementation(libs.antlr4.runtime)
}
tasks.withType(Test::class.java).configureEach {
@@ -171,35 +147,4 @@ tasks.named<Test>("intTest").configure {
// This property is not honored in a per-profile application.properties file,
// so we need to set it here.
systemProperty("quarkus.log.file.path",
logsDir.resolve("polaris.log").absolutePath)
- // For Spark integration tests
- addSparkJvmOptions()
-}
-
-/**
- * Adds the JPMS options required for Spark to run on Java 17, taken from the
- * `DEFAULT_MODULE_OPTIONS` constant in
`org.apache.spark.launcher.JavaModuleOptions`.
- */
-fun JavaForkOptions.addSparkJvmOptions() {
- jvmArgs =
- (jvmArgs ?: emptyList()) +
- listOf(
- // Spark 3.3+
- "-XX:+IgnoreUnrecognizedVMOptions",
- "--add-opens=java.base/java.lang=ALL-UNNAMED",
- "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED",
- "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED",
- "--add-opens=java.base/java.io=ALL-UNNAMED",
- "--add-opens=java.base/java.net=ALL-UNNAMED",
- "--add-opens=java.base/java.nio=ALL-UNNAMED",
- "--add-opens=java.base/java.util=ALL-UNNAMED",
- "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED",
- "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED",
- "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
- "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED",
- "--add-opens=java.base/sun.security.action=ALL-UNNAMED",
- "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED",
- "--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED",
- // Spark 3.4+
- "-Djdk.reflect.useDirectMethodHandle=false",
- )
}
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java
index 84ac558b..fb23d5f4 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java
@@ -34,6 +34,7 @@ import io.quarkus.test.junit.QuarkusMock;
import io.quarkus.test.junit.QuarkusTest;
import io.quarkus.test.junit.QuarkusTestProfile;
import io.quarkus.test.junit.TestProfile;
+import jakarta.annotation.Nonnull;
import jakarta.inject.Inject;
import jakarta.ws.rs.core.SecurityContext;
import java.io.IOException;
@@ -122,7 +123,6 @@ import org.apache.polaris.service.types.NotificationRequest;
import org.apache.polaris.service.types.NotificationType;
import org.apache.polaris.service.types.TableUpdateNotification;
import org.assertj.core.api.Assertions;
-import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
@@ -1630,13 +1630,13 @@ public class BasePolarisCatalogTest extends
CatalogTests<BasePolarisCatalog> {
new FileIOFactory() {
@Override
public FileIO loadFileIO(
- @NotNull CallContext callContext,
- @NotNull String ioImplClassName,
- @NotNull Map<String, String> properties,
- @NotNull TableIdentifier identifier,
- @NotNull Set<String> tableLocations,
- @NotNull Set<PolarisStorageActions> storageActions,
- @NotNull PolarisResolvedPathWrapper resolvedEntityPath) {
+ @Nonnull CallContext callContext,
+ @Nonnull String ioImplClassName,
+ @Nonnull Map<String, String> properties,
+ @Nonnull TableIdentifier identifier,
+ @Nonnull Set<String> tableLocations,
+ @Nonnull Set<PolarisStorageActions> storageActions,
+ @Nonnull PolarisResolvedPathWrapper resolvedEntityPath) {
return measured.loadFileIO(
callContext,
"org.apache.iceberg.inmemory.InMemoryFileIO",
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java
index b6c03ef8..f850b08f 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java
@@ -23,6 +23,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatPredicate;
import io.quarkus.test.junit.QuarkusTest;
+import jakarta.annotation.Nonnull;
import jakarta.inject.Inject;
import java.io.IOException;
import java.util.HashMap;
@@ -62,7 +63,6 @@ import org.apache.polaris.service.catalog.io.FileIOFactory;
import org.apache.polaris.service.task.ManifestFileCleanupTaskHandler;
import org.apache.polaris.service.task.TaskFileIOSupplier;
import org.apache.polaris.service.task.TaskUtils;
-import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.Test;
@QuarkusTest
@@ -76,13 +76,13 @@ class ManifestFileCleanupTaskHandlerTest {
new FileIOFactory() {
@Override
public FileIO loadFileIO(
- @NotNull CallContext callContext,
- @NotNull String ioImplClassName,
- @NotNull Map<String, String> properties,
- @NotNull TableIdentifier identifier,
- @NotNull Set<String> tableLocations,
- @NotNull Set<PolarisStorageActions> storageActions,
- @NotNull PolarisResolvedPathWrapper resolvedEntityPath) {
+ @Nonnull CallContext callContext,
+ @Nonnull String ioImplClassName,
+ @Nonnull Map<String, String> properties,
+ @Nonnull TableIdentifier identifier,
+ @Nonnull Set<String> tableLocations,
+ @Nonnull Set<PolarisStorageActions> storageActions,
+ @Nonnull PolarisResolvedPathWrapper resolvedEntityPath) {
return fileIO;
}
});
diff --git a/quarkus/spark-tests/build.gradle.kts
b/quarkus/spark-tests/build.gradle.kts
new file mode 100644
index 00000000..18cc9d58
--- /dev/null
+++ b/quarkus/spark-tests/build.gradle.kts
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+ alias(libs.plugins.quarkus)
+ alias(libs.plugins.jandex)
+ id("polaris-quarkus")
+}
+
+dependencies {
+
+ // must be enforced to get a consistent and validated set of dependencies
+ implementation(enforcedPlatform(libs.quarkus.bom)) {
+ exclude(group = "org.antlr", module = "antlr4-runtime")
+ exclude(group = "org.scala-lang", module = "scala-library")
+ exclude(group = "org.scala-lang", module = "scala-reflect")
+ }
+
+ implementation(project(":polaris-quarkus-service"))
+
+ testImplementation(project(":polaris-tests"))
+ testImplementation(testFixtures(project(":polaris-quarkus-service")))
+
+ testImplementation(platform(libs.quarkus.bom))
+ testImplementation("io.quarkus:quarkus-junit5")
+ testImplementation("io.quarkus:quarkus-rest-client")
+ testImplementation("io.quarkus:quarkus-rest-client-jackson")
+
+ testImplementation(platform(libs.awssdk.bom))
+ testImplementation("software.amazon.awssdk:glue")
+ testImplementation("software.amazon.awssdk:kms")
+ testImplementation("software.amazon.awssdk:dynamodb")
+
+ testImplementation(platform(libs.testcontainers.bom))
+ testImplementation("org.testcontainers:testcontainers")
+ testImplementation(libs.s3mock.testcontainers)
+
+ // Required for Spark integration tests
+ testImplementation(enforcedPlatform(libs.scala212.lang.library))
+ testImplementation(enforcedPlatform(libs.scala212.lang.reflect))
+ testImplementation(libs.javax.servlet.api)
+ testImplementation(libs.antlr4.runtime)
+}
+
+tasks.named<Test>("intTest").configure {
+ maxParallelForks = 1
+ systemProperty("java.util.logging.manager",
"org.jboss.logmanager.LogManager")
+ if (System.getenv("AWS_REGION") == null) {
+ environment("AWS_REGION", "us-west-2")
+ }
+ // Note: the test secrets are referenced in
+ // org.apache.polaris.service.quarkus.it.QuarkusServerManager
+ environment("POLARIS_BOOTSTRAP_CREDENTIALS",
"POLARIS,test-admin,test-secret")
+ jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED")
+ // Need to allow a java security manager after Java 21, for
Subject.getSubject to work
+ // "getSubject is supported only if a security manager is allowed".
+ systemProperty("java.security.manager", "allow")
+ // Same issue as above: allow a java security manager after Java 21
+ // (this setting is for the application under test, while the setting above
is for test code).
+ systemProperty("quarkus.test.arg-line", "-Djava.security.manager=allow")
+ val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs")
+ // delete files from previous runs
+ doFirst {
+ // delete log files written by Polaris
+ logsDir.deleteRecursively()
+ // delete quarkus.log file (captured Polaris stdout/stderr)
+ project.layout.buildDirectory.get().asFile.resolve("quarkus.log").delete()
+ }
+ // This property is not honored in a per-profile application.properties file,
+ // so we need to set it here.
+ systemProperty("quarkus.log.file.path",
logsDir.resolve("polaris.log").absolutePath)
+ // For Spark integration tests
+ addSparkJvmOptions()
+}
+
+/**
+ * Adds the JPMS options required for Spark to run on Java 17, taken from the
+ * `DEFAULT_MODULE_OPTIONS` constant in
`org.apache.spark.launcher.JavaModuleOptions`.
+ */
+fun JavaForkOptions.addSparkJvmOptions() {
+ jvmArgs =
+ (jvmArgs ?: emptyList()) +
+ listOf(
+ // Spark 3.3+
+ "-XX:+IgnoreUnrecognizedVMOptions",
+ "--add-opens=java.base/java.lang=ALL-UNNAMED",
+ "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED",
+ "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED",
+ "--add-opens=java.base/java.io=ALL-UNNAMED",
+ "--add-opens=java.base/java.net=ALL-UNNAMED",
+ "--add-opens=java.base/java.nio=ALL-UNNAMED",
+ "--add-opens=java.base/java.util=ALL-UNNAMED",
+ "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED",
+ "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED",
+ "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
+ "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED",
+ "--add-opens=java.base/sun.security.action=ALL-UNNAMED",
+ "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED",
+ "--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED",
+ // Spark 3.4+
+ "-Djdk.reflect.useDirectMethodHandle=false",
+ )
+}
diff --git
a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/QuarkusSparkIT.java
b/quarkus/spark-tests/src/intTest/java/org/apache/polaris/service/quarkus/spark/it/QuarkusSparkIT.java
similarity index 86%
rename from
quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/QuarkusSparkIT.java
rename to
quarkus/spark-tests/src/intTest/java/org/apache/polaris/service/quarkus/spark/it/QuarkusSparkIT.java
index 052eebf5..a3fd5345 100644
---
a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/QuarkusSparkIT.java
+++
b/quarkus/spark-tests/src/intTest/java/org/apache/polaris/service/quarkus/spark/it/QuarkusSparkIT.java
@@ -16,12 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.polaris.service.quarkus.it;
+package org.apache.polaris.service.quarkus.spark.it;
import io.quarkus.test.junit.QuarkusIntegrationTest;
-import io.quarkus.test.junit.QuarkusTestProfile;
-import io.quarkus.test.junit.TestProfile;
-import java.util.Map;
import org.apache.polaris.service.it.test.PolarisSparkIntegrationTest;
@QuarkusIntegrationTest
diff --git
a/quarkus/spark-tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager
b/quarkus/spark-tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager
new file mode 100644
index 00000000..a8615d99
--- /dev/null
+++
b/quarkus/spark-tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+org.apache.polaris.service.quarkus.it.QuarkusServerManager
\ No newline at end of file
