This is an automated email from the ASF dual-hosted git repository.
dimas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new ace94a5a7 NoSQL: Add to runtime-service (#3396)
ace94a5a7 is described below
commit ace94a5a7842913c95c29c265cf4ff3d3ed3e8fe
Author: Robert Stupp <[email protected]>
AuthorDate: Tue Jan 27 20:39:03 2026 +0100
NoSQL: Add to runtime-service (#3396)
* NoSQL: Add to runtime-service
This change adds the NoSQL persistence to polaris-runtime-service.
---
.../src/main/resources/application-test.properties | 1 +
.../src/main/resources/application.properties | 21 +-
runtime/service/build.gradle.kts | 10 +
.../service/it/nosql/NoSqlApplicationIT.java} | 31 +--
.../polaris/service/it/nosql/NoSqlCatalogIT.java} | 33 ++--
.../it/nosql/NoSqlManagementServiceIT.java} | 31 +--
.../polaris/service/it/nosql/NoSqlTesting.java} | 25 +--
.../apache/polaris/service/catalog/Profiles.java | 17 ++
.../PolarisGenericTableCatalogNoSqlInMemTest.java | 44 +++++
.../IcebergCatalogHandlerNoSqlAuthzTest.java | 53 +++++
.../iceberg/IcebergCatalogNoSqlInMemTest.java | 64 ++++++
.../iceberg/IcebergViewCatalogNoSqlInMemTest.java | 56 ++++++
.../policy/PolicyCatalogNoSqlInMemTest.java | 43 +++++
.../polaris/service/distcache/HttpTestServer.java | 70 +++++++
...rsistenceDistCacheInvalidationsIntegration.java | 214 +++++++++++++++++++++
15 files changed, 630 insertions(+), 83 deletions(-)
diff --git a/runtime/defaults/src/main/resources/application-test.properties
b/runtime/defaults/src/main/resources/application-test.properties
index 0e6aecedf..abd7d8e7d 100644
--- a/runtime/defaults/src/main/resources/application-test.properties
+++ b/runtime/defaults/src/main/resources/application-test.properties
@@ -22,6 +22,7 @@
quarkus.datasource.devservices.enabled=false
quarkus.keycloak.devservices.enabled=false
+quarkus.mongodb.devservices.enabled=false
quarkus.log.level=ERROR
quarkus.log.file.enable=false
diff --git a/runtime/defaults/src/main/resources/application.properties
b/runtime/defaults/src/main/resources/application.properties
index aa4fe4ad1..f05048a35 100644
--- a/runtime/defaults/src/main/resources/application.properties
+++ b/runtime/defaults/src/main/resources/application.properties
@@ -41,6 +41,8 @@ quarkus.micrometer.enabled=true
quarkus.micrometer.export.prometheus.enabled=true
quarkus.oidc.enabled=true
quarkus.otel.enabled=true
+#quarkus.mongodb.metrics.enabled=true
+#quarkus.mongodb.connection-string=mongodb://localhost:27017
# ---- Runtime Configuration ----
# Below are default values for properties that can be changed in runtime.
@@ -128,9 +130,19 @@
polaris.features."SUPPORTED_CATALOG_CONNECTION_TYPES"=["ICEBERG_REST"]
# realm overrides
#
polaris.features.realm-overrides."my-realm"."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true
-# polaris.persistence.type=in-memory-atomic
+# Available types:
+# - in-memory - InMemoryPolarisMetaStoreManagerFactory
+# - in-memory-atomic - InMemoryAtomicOperationMetaStoreManagerFactory
+# - nosql (beta) - NoSQL persistence backend, define the backend type via
'polaris.persistence.nosql.backend'
+# - relational-jdbc
polaris.persistence.type=in-memory
-# polaris.persistence.type=relational-jdbc
+# Database backend for 'nosql' persistence-type
+# Available backends:
+# - InMemory - for testing purposes
+# - MongoDb - configure the via the Quarkus extension
+# Configure the necessary MongoDB properties starting with
'quarkus.mongodb.' in this file.
+# See https://quarkus.io/guides/mongodb#configuration-reference for
details about these configurations.
+#polaris.persistence.nosql.backend=InMemory
polaris.secrets-manager.type=in-memory
# if set to true it will try to start localstack at build and run time for the
local environment
@@ -138,6 +150,9 @@ polaris.secrets-manager.type=in-memory
quarkus.rds.devservices.enabled=false
quarkus.rds.sync-client.type=apache
+## MongoDB specific configuration
+#quarkus.mongodb.database=polaris
+
polaris.file-io.type=default
polaris.event-listener.type=no-op
@@ -282,6 +297,8 @@ quarkus.arc.ignored-split-packages=\
## Quarkus required setting for third party indexing
# fixed at build-time
+quarkus.index-dependency.agrona.group-id=org.agrona
+quarkus.index-dependency.agrona.artifact-id=agrona
quarkus.index-dependency.avro.group-id=org.apache.avro
quarkus.index-dependency.avro.artifact-id=avro
quarkus.index-dependency.guava.group-id=com.google.guava
diff --git a/runtime/service/build.gradle.kts b/runtime/service/build.gradle.kts
index 9dcd8c588..11d1b7a48 100644
--- a/runtime/service/build.gradle.kts
+++ b/runtime/service/build.gradle.kts
@@ -39,6 +39,12 @@ dependencies {
compileOnly(project(":polaris-immutables"))
annotationProcessor(project(":polaris-immutables", configuration =
"processor"))
+ runtimeOnly(project(":polaris-persistence-nosql-metastore"))
+ runtimeOnly(project(":polaris-persistence-nosql-cdi-quarkus"))
+ runtimeOnly(project(":polaris-persistence-nosql-cdi-quarkus-distcache"))
+ runtimeOnly(project(":polaris-persistence-nosql-maintenance-impl"))
+ runtimeOnly(project(":polaris-persistence-nosql-metastore-maintenance"))
+
implementation(platform(libs.iceberg.bom))
implementation("org.apache.iceberg:iceberg-api")
implementation("org.apache.iceberg:iceberg-core")
@@ -153,6 +159,10 @@ dependencies {
testImplementation("org.testcontainers:testcontainers")
testImplementation("org.testcontainers:testcontainers-postgresql")
+ testImplementation(project(":polaris-persistence-nosql-api"))
+ testImplementation(testFixtures(project(":polaris-persistence-nosql-api")))
+ testImplementation(project(":polaris-persistence-nosql-impl"))
+
testFixturesImplementation(project(":polaris-core"))
testFixturesImplementation(project(":polaris-api-management-model"))
testFixturesImplementation(project(":polaris-api-management-service"))
diff --git
a/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java
b/runtime/service/src/intTest/java/org/apache/polaris/service/it/nosql/NoSqlApplicationIT.java
similarity index 52%
copy from
runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java
copy to
runtime/service/src/intTest/java/org/apache/polaris/service/it/nosql/NoSqlApplicationIT.java
index c4231ac26..0181296f0 100644
---
a/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java
+++
b/runtime/service/src/intTest/java/org/apache/polaris/service/it/nosql/NoSqlApplicationIT.java
@@ -16,29 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
+package org.apache.polaris.service.it.nosql;
-package org.apache.polaris.service.catalog;
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import io.quarkus.test.junit.TestProfile;
+import org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest;
-import io.quarkus.test.junit.QuarkusTestProfile;
-import java.util.Map;
-
-public final class Profiles {
- private Profiles() {}
-
- public static class DefaultProfile implements QuarkusTestProfile {
- @Override
- public Map<String, String> getConfigOverrides() {
- return Map.of(
- "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"",
- "true",
- "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
- "true",
- "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
- "[\"FILE\",\"S3\"]",
- "polaris.event-listener.type",
- "test",
- "polaris.readiness.ignore-severe-issues",
- "true");
- }
- }
-}
+@QuarkusIntegrationTest
+@TestProfile(value = NoSqlTesting.PersistenceInMemoryProfile.class)
+public class NoSqlApplicationIT extends PolarisApplicationIntegrationTest {}
diff --git
a/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java
b/runtime/service/src/intTest/java/org/apache/polaris/service/it/nosql/NoSqlCatalogIT.java
similarity index 52%
copy from
runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java
copy to
runtime/service/src/intTest/java/org/apache/polaris/service/it/nosql/NoSqlCatalogIT.java
index c4231ac26..ba99caab5 100644
---
a/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java
+++
b/runtime/service/src/intTest/java/org/apache/polaris/service/it/nosql/NoSqlCatalogIT.java
@@ -16,29 +16,26 @@
* specific language governing permissions and limitations
* under the License.
*/
+package org.apache.polaris.service.it.nosql;
-package org.apache.polaris.service.catalog;
-
-import io.quarkus.test.junit.QuarkusTestProfile;
+import com.google.common.collect.ImmutableMap;
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import io.quarkus.test.junit.TestProfile;
import java.util.Map;
+import org.apache.polaris.service.it.PolarisRestCatalogMinIOIT;
-public final class Profiles {
- private Profiles() {}
-
- public static class DefaultProfile implements QuarkusTestProfile {
+@QuarkusIntegrationTest
+@TestProfile(value = NoSqlCatalogIT.Profile.class)
+public class NoSqlCatalogIT extends PolarisRestCatalogMinIOIT {
+ public static class Profile extends NoSqlTesting.PersistenceInMemoryProfile {
@Override
public Map<String, String> getConfigOverrides() {
- return Map.of(
- "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"",
- "true",
- "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
- "true",
- "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
- "[\"FILE\",\"S3\"]",
- "polaris.event-listener.type",
- "test",
- "polaris.readiness.ignore-severe-issues",
- "true");
+ return ImmutableMap.<String, String>builder()
+ .putAll(super.getConfigOverrides())
+ .put("polaris.storage.aws.access-key", MINIO_ACCESS_KEY)
+ .put("polaris.storage.aws.secret-key", MINIO_SECRET_KEY)
+ .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"",
"false")
+ .build();
}
}
}
diff --git
a/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java
b/runtime/service/src/intTest/java/org/apache/polaris/service/it/nosql/NoSqlManagementServiceIT.java
similarity index 52%
copy from
runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java
copy to
runtime/service/src/intTest/java/org/apache/polaris/service/it/nosql/NoSqlManagementServiceIT.java
index c4231ac26..c76cf4b87 100644
---
a/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java
+++
b/runtime/service/src/intTest/java/org/apache/polaris/service/it/nosql/NoSqlManagementServiceIT.java
@@ -16,29 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
+package org.apache.polaris.service.it.nosql;
-package org.apache.polaris.service.catalog;
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import io.quarkus.test.junit.TestProfile;
+import
org.apache.polaris.service.it.test.PolarisManagementServiceIntegrationTest;
-import io.quarkus.test.junit.QuarkusTestProfile;
-import java.util.Map;
-
-public final class Profiles {
- private Profiles() {}
-
- public static class DefaultProfile implements QuarkusTestProfile {
- @Override
- public Map<String, String> getConfigOverrides() {
- return Map.of(
- "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"",
- "true",
- "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
- "true",
- "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
- "[\"FILE\",\"S3\"]",
- "polaris.event-listener.type",
- "test",
- "polaris.readiness.ignore-severe-issues",
- "true");
- }
- }
-}
+@QuarkusIntegrationTest
+@TestProfile(value = NoSqlTesting.PersistenceInMemoryProfile.class)
+public class NoSqlManagementServiceIT extends
PolarisManagementServiceIntegrationTest {}
diff --git
a/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java
b/runtime/service/src/intTest/java/org/apache/polaris/service/it/nosql/NoSqlTesting.java
similarity index 63%
copy from
runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java
copy to
runtime/service/src/intTest/java/org/apache/polaris/service/it/nosql/NoSqlTesting.java
index c4231ac26..b8bc044e8 100644
---
a/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java
+++
b/runtime/service/src/intTest/java/org/apache/polaris/service/it/nosql/NoSqlTesting.java
@@ -16,29 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-
-package org.apache.polaris.service.catalog;
+package org.apache.polaris.service.it.nosql;
import io.quarkus.test.junit.QuarkusTestProfile;
import java.util.Map;
-public final class Profiles {
- private Profiles() {}
+public final class NoSqlTesting {
+ private NoSqlTesting() {}
- public static class DefaultProfile implements QuarkusTestProfile {
+ public static class PersistenceInMemoryProfile implements QuarkusTestProfile
{
@Override
public Map<String, String> getConfigOverrides() {
return Map.of(
- "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"",
- "true",
- "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"",
- "true",
- "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
- "[\"FILE\",\"S3\"]",
- "polaris.event-listener.type",
- "test",
- "polaris.readiness.ignore-severe-issues",
- "true");
+ "polaris.persistence.type",
+ "nosql",
+ "polaris.persistence.nosql.backend",
+ "InMemory",
+ "polaris.persistence.auto-bootstrap-types",
+ "nosql");
}
}
}
diff --git
a/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java
b/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java
index c4231ac26..9c5411037 100644
---
a/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java
+++
b/runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java
@@ -19,12 +19,19 @@
package org.apache.polaris.service.catalog;
+import com.google.common.collect.ImmutableMap;
import io.quarkus.test.junit.QuarkusTestProfile;
import java.util.Map;
public final class Profiles {
private Profiles() {}
+ public static final Map<String, String> NOSQL_IN_MEM =
+ ImmutableMap.<String, String>builder()
+ .put("polaris.persistence.type", "nosql")
+ .put("polaris.persistence.nosql.backend", "InMemory")
+ .build();
+
public static class DefaultProfile implements QuarkusTestProfile {
@Override
public Map<String, String> getConfigOverrides() {
@@ -41,4 +48,14 @@ public final class Profiles {
"true");
}
}
+
+ public static class DefaultNoSqlProfile extends DefaultProfile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return ImmutableMap.<String, String>builder()
+ .putAll(super.getConfigOverrides())
+ .putAll(NOSQL_IN_MEM)
+ .build();
+ }
+ }
}
diff --git
a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogNoSqlInMemTest.java
b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogNoSqlInMemTest.java
new file mode 100644
index 000000000..a7caa216f
--- /dev/null
+++
b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogNoSqlInMemTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.catalog.generic;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.inject.Inject;
+import java.util.List;
+import org.apache.polaris.core.persistence.MetaStoreManagerFactory;
+import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet;
+import org.apache.polaris.service.catalog.Profiles;
+
+@QuarkusTest
+@TestProfile(Profiles.DefaultNoSqlProfile.class)
+public class PolarisGenericTableCatalogNoSqlInMemTest
+ extends AbstractPolarisGenericTableCatalogTest {
+
+ @Inject MetaStoreManagerFactory metaStoreManagerFactory;
+
+ @Override
+ protected void bootstrapRealm(String realmName) {
+ metaStoreManagerFactory
+ .bootstrapRealms(
+ List.of(realmName),
+ RootCredentialsSet.fromList(List.of(realmName +
",aClientId,aSecret")))
+ .get(realmName);
+ }
+}
diff --git
a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerNoSqlAuthzTest.java
b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerNoSqlAuthzTest.java
new file mode 100644
index 000000000..aad67ce27
--- /dev/null
+++
b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerNoSqlAuthzTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.catalog.iceberg;
+
+import static org.apache.polaris.service.catalog.Profiles.NOSQL_IN_MEM;
+
+import com.google.common.collect.ImmutableMap;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.inject.Inject;
+import java.util.List;
+import java.util.Map;
+import org.apache.polaris.core.persistence.MetaStoreManagerFactory;
+import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet;
+import org.apache.polaris.service.admin.PolarisAuthzTestBase;
+
+@QuarkusTest
+@TestProfile(IcebergCatalogHandlerNoSqlAuthzTest.Profile.class)
+public class IcebergCatalogHandlerNoSqlAuthzTest extends
AbstractIcebergCatalogHandlerAuthzTest {
+
+ public static class Profile extends PolarisAuthzTestBase.Profile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return ImmutableMap.<String, String>builder()
+ .putAll(super.getConfigOverrides())
+ .putAll(NOSQL_IN_MEM)
+ .build();
+ }
+ }
+
+ @Inject MetaStoreManagerFactory metaStoreManagerFactory;
+
+ @Override
+ protected void bootstrapRealm(String realmIdentifier) {
+ metaStoreManagerFactory.bootstrapRealms(List.of(realmIdentifier),
RootCredentialsSet.EMPTY);
+ }
+}
diff --git
a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogNoSqlInMemTest.java
b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogNoSqlInMemTest.java
new file mode 100644
index 000000000..536c17c03
--- /dev/null
+++
b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogNoSqlInMemTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.catalog.iceberg;
+
+import static org.apache.polaris.service.catalog.Profiles.NOSQL_IN_MEM;
+
+import com.google.common.collect.ImmutableMap;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.TestProfile;
+import java.util.List;
+import java.util.Map;
+import org.apache.polaris.core.PolarisDiagnostics;
+import org.apache.polaris.core.config.RealmConfig;
+import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
+import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet;
+import org.apache.polaris.core.persistence.cache.EntityCache;
+
+@QuarkusTest
+@TestProfile(IcebergCatalogNoSqlInMemTest.Profile.class)
+public class IcebergCatalogNoSqlInMemTest extends AbstractIcebergCatalogTest {
+
+ public static class Profile extends AbstractIcebergCatalogTest.Profile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return ImmutableMap.<String, String>builder()
+ .putAll(super.getConfigOverrides())
+ .putAll(NOSQL_IN_MEM)
+ .build();
+ }
+ }
+
+ @Override
+ protected void bootstrapRealm(String realmName) {
+ metaStoreManagerFactory
+ .bootstrapRealms(
+ List.of(realmName),
+ RootCredentialsSet.fromList(List.of(realmName +
",aClientId,aSecret")))
+ .get(realmName);
+ }
+
+ @Override
+ protected EntityCache createEntityCache(
+ PolarisDiagnostics diagnostics,
+ RealmConfig realmConfig,
+ PolarisMetaStoreManager metaStoreManager) {
+ return null;
+ }
+}
diff --git
a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergViewCatalogNoSqlInMemTest.java
b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergViewCatalogNoSqlInMemTest.java
new file mode 100644
index 000000000..4089c173f
--- /dev/null
+++
b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergViewCatalogNoSqlInMemTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.catalog.iceberg;
+
+import static org.apache.polaris.service.catalog.Profiles.NOSQL_IN_MEM;
+
+import com.google.common.collect.ImmutableMap;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.inject.Inject;
+import java.util.List;
+import java.util.Map;
+import org.apache.polaris.core.persistence.MetaStoreManagerFactory;
+import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet;
+
+@QuarkusTest
+@TestProfile(IcebergViewCatalogNoSqlInMemTest.Profile.class)
+public class IcebergViewCatalogNoSqlInMemTest extends
AbstractIcebergCatalogViewTest {
+
+ @Inject MetaStoreManagerFactory metaStoreManagerFactory;
+
+ public static class Profile extends AbstractIcebergCatalogViewTest.Profile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return ImmutableMap.<String, String>builder()
+ .putAll(super.getConfigOverrides())
+ .putAll(NOSQL_IN_MEM)
+ .build();
+ }
+ }
+
+ @Override
+ protected void bootstrapRealm(String realmName) {
+ metaStoreManagerFactory
+ .bootstrapRealms(
+ List.of(realmName),
+ RootCredentialsSet.fromList(List.of(realmName +
",aClientId,aSecret")))
+ .get(realmName);
+ }
+}
diff --git
a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogNoSqlInMemTest.java
b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogNoSqlInMemTest.java
new file mode 100644
index 000000000..521769e4b
--- /dev/null
+++
b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/PolicyCatalogNoSqlInMemTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.catalog.policy;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.inject.Inject;
+import java.util.List;
+import org.apache.polaris.core.persistence.MetaStoreManagerFactory;
+import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet;
+import org.apache.polaris.service.catalog.Profiles;
+
+@QuarkusTest
+@TestProfile(Profiles.DefaultNoSqlProfile.class)
+public class PolicyCatalogNoSqlInMemTest extends AbstractPolicyCatalogTest {
+
+ @Inject MetaStoreManagerFactory metaStoreManagerFactory;
+
+ @Override
+ protected void bootstrapRealm(String realmName) {
+ metaStoreManagerFactory
+ .bootstrapRealms(
+ List.of(realmName),
+ RootCredentialsSet.fromList(List.of(realmName +
",aClientId,aSecret")))
+ .get(realmName);
+ }
+}
diff --git
a/runtime/service/src/test/java/org/apache/polaris/service/distcache/HttpTestServer.java
b/runtime/service/src/test/java/org/apache/polaris/service/distcache/HttpTestServer.java
new file mode 100644
index 000000000..d89ef619a
--- /dev/null
+++
b/runtime/service/src/test/java/org/apache/polaris/service/distcache/HttpTestServer.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.distcache;
+
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+
+/** HTTP test server. */
+public class HttpTestServer implements AutoCloseable {
+ private final HttpServer server;
+
+ public HttpTestServer(String context, HttpHandler handler) throws
IOException {
+ this(new InetSocketAddress("localhost", 0), context, handler);
+ }
+
+ public HttpTestServer(InetSocketAddress bind, String context, HttpHandler
handler)
+ throws IOException {
+ HttpHandler safeHandler =
+ exchange -> {
+ try {
+ handler.handle(exchange);
+ } catch (RuntimeException | Error e) {
+ exchange.sendResponseHeaders(503, 0);
+ throw e;
+ }
+ };
+ server = HttpServer.create(bind, 0);
+ server.createContext(context, safeHandler);
+ server.setExecutor(null);
+
+ server.start();
+ }
+
+ public InetSocketAddress getAddress() {
+ return server.getAddress();
+ }
+
+ public URI getUri() {
+ return URI.create(
+ "http://";
+ + getAddress().getAddress().getHostAddress()
+ + ":"
+ + getAddress().getPort()
+ + "/");
+ }
+
+ @Override
+ public void close() {
+ server.stop(0);
+ }
+}
diff --git
a/runtime/service/src/test/java/org/apache/polaris/service/distcache/TestPersistenceDistCacheInvalidationsIntegration.java
b/runtime/service/src/test/java/org/apache/polaris/service/distcache/TestPersistenceDistCacheInvalidationsIntegration.java
new file mode 100644
index 000000000..9b8cc63b4
--- /dev/null
+++
b/runtime/service/src/test/java/org/apache/polaris/service/distcache/TestPersistenceDistCacheInvalidationsIntegration.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.distcache;
+
+import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON;
+import static java.lang.String.format;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Map.entry;
+import static java.util.Objects.requireNonNull;
+import static
org.apache.polaris.persistence.nosql.api.cache.CacheInvalidations.CacheInvalidationEvictObj.cacheInvalidationEvictObj;
+import static
org.apache.polaris.persistence.nosql.api.cache.CacheInvalidations.CacheInvalidationEvictReference.cacheInvalidationEvictReference;
+import static org.apache.polaris.persistence.nosql.api.obj.ObjRef.objRef;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.InstanceOfAssertFactories.list;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableMap;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.inject.Inject;
+import java.io.InputStream;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import org.apache.polaris.persistence.nosql.api.Persistence;
+import org.apache.polaris.persistence.nosql.api.RealmPersistenceFactory;
+import org.apache.polaris.persistence.nosql.api.SystemPersistence;
+import org.apache.polaris.persistence.nosql.api.cache.CacheBackend;
+import org.apache.polaris.persistence.nosql.api.cache.CacheInvalidations;
+import
org.apache.polaris.persistence.nosql.api.cache.CacheInvalidations.CacheInvalidation;
+import org.apache.polaris.persistence.nosql.api.obj.SimpleTestObj;
+import org.apache.polaris.service.catalog.iceberg.AbstractIcebergCatalogTest;
+import org.assertj.core.api.SoftAssertions;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledOnOs;
+import org.junit.jupiter.api.condition.OS;
+
+@QuarkusTest
+@TestProfile(TestPersistenceDistCacheInvalidationsIntegration.Profile.class)
+// Need two IPs in this test. macOS doesn't allow binding to arbitrary
127.x.x.x addresses.
+@EnabledOnOs(OS.LINUX)
+@SuppressWarnings("CdiInjectionPointsInspection")
+public class TestPersistenceDistCacheInvalidationsIntegration {
+
+ public static class Profile extends AbstractIcebergCatalogTest.Profile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return ImmutableMap.<String, String>builder()
+ .putAll(super.getConfigOverrides())
+ .put("quarkus.management.port", "" + QUARKUS_MANAGEMENT_PORT)
+ .put("quarkus.management.test-port", "" + QUARKUS_MANAGEMENT_PORT)
+ .put("quarkus.management.host", "127.0.0.1")
+ .put("quarkus.management.enabled", "true")
+ .put("polaris.persistence.type", "nosql")
+ .put("polaris.persistence.auto-bootstrap-types", "nosql")
+ .put("polaris.persistence.nosql.backend", "InMemory")
+ .put(
+
"polaris.persistence.distributed-cache-invalidations.valid-tokens", "token1," +
TOKEN)
+ .put(
+
"polaris.persistence.distributed-cache-invalidations.service-names",
+ "=127.0.0.1,=127.1.2.3")
+ .build();
+ }
+ }
+
+ static final String TOKEN = "otherToken";
+ static final String ENDPOINT = "/polaris-management/cache-coherency";
+
+ // MUST be constant for test AND service
+ static final int QUARKUS_MANAGEMENT_PORT = 64321;
+
+ static URI CACHE_INVALIDATIONS_ENDPOINT =
+ URI.create(
+ format(
+ "http://127.0.0.1:%d%s?sender="; + UUID.randomUUID(),
+ QUARKUS_MANAGEMENT_PORT,
+ ENDPOINT));
+
+ SoftAssertions soft;
+ ObjectMapper mapper;
+
+ @Inject CacheBackend cacheBackend;
+
+ @Inject @SystemPersistence Persistence systemPersistence;
+ @Inject RealmPersistenceFactory realmPersistenceFactory;
+
+ @BeforeEach
+ public void before() {
+ soft = new SoftAssertions();
+ mapper = new ObjectMapper();
+ }
+
+ @AfterEach
+ public void after() {
+ soft.assertAll();
+ }
+
+ @Test
+ public void systemRealm() throws Exception {
+ sendReceive(systemPersistence);
+ }
+
+ @Test
+ public void otherRealm() throws Exception {
+ var persistence =
realmPersistenceFactory.newBuilder().realmId("foo").build();
+ sendReceive(persistence);
+ }
+
+ private void sendReceive(Persistence persistence) throws Exception {
+ var queue = new LinkedBlockingQueue<Map.Entry<URI, String>>();
+ try (var ignore =
+ new HttpTestServer(
+ new InetSocketAddress("127.1.2.3", QUARKUS_MANAGEMENT_PORT),
+ ENDPOINT,
+ exchange -> {
+ try (InputStream requestBody = exchange.getRequestBody()) {
+ queue.add(
+ entry(exchange.getRequestURI(), new
String(requestBody.readAllBytes(), UTF_8)));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ exchange.sendResponseHeaders(204, -1);
+ exchange.getResponseBody().close();
+ })) {
+
+ var obj =
SimpleTestObj.builder().id(persistence.generateId()).text("test").build();
+ persistence.write(obj, SimpleTestObj.class);
+
+ // verify that "we" received the invalidation for the obj
+ var invalidation = queue.poll(1, TimeUnit.MINUTES);
+ var uri = requireNonNull(invalidation).getKey();
+ soft.assertThat(uri.getRawQuery()).startsWith("sender=");
+ var invalidations = mapper.readValue(invalidation.getValue(),
CacheInvalidations.class);
+ soft.assertThat(invalidations)
+ .extracting(CacheInvalidations::invalidations,
list(CacheInvalidation.class))
+ .containsExactly(cacheInvalidationEvictObj(persistence.realmId(),
objRef(obj)));
+
+ // verify that "the service" processes an invalidation
+ soft.assertThat(cacheBackend.get(persistence.realmId(),
objRef(obj))).isNotNull();
+ send(invalidations);
+ awaitCondition(
+ () -> assertThat(cacheBackend.get(persistence.realmId(),
objRef(obj))).isNull());
+
+ // reference
+
+ var refName = "foo-ref";
+ persistence.createReference(refName, Optional.empty());
+
+ // verify that "we" received the invalidation for the reference
+ invalidation = queue.poll(1, TimeUnit.MINUTES);
+ uri = requireNonNull(invalidation).getKey();
+ soft.assertThat(uri.getRawQuery()).startsWith("sender=");
+ invalidations = mapper.readValue(invalidation.getValue(),
CacheInvalidations.class);
+ soft.assertThat(invalidations)
+ .extracting(CacheInvalidations::invalidations,
list(CacheInvalidation.class))
+
.containsExactly(cacheInvalidationEvictReference(persistence.realmId(),
refName));
+
+ // verify that "the service" processes an invalidation
+ soft.assertThat(cacheBackend.getReference(persistence.realmId(),
refName)).isNotNull();
+ send(invalidations);
+ awaitCondition(
+ () -> assertThat(cacheBackend.get(persistence.realmId(),
objRef(obj))).isNull());
+ }
+ }
+
+ @SuppressWarnings("BusyWait")
+ private void awaitCondition(Runnable test) throws Exception {
+ var tEnd = System.nanoTime() + TimeUnit.MINUTES.toNanos(1);
+ while (System.nanoTime() < tEnd) {
+ try {
+ test.run();
+ return;
+ } catch (AssertionError e) {
+ if (System.nanoTime() > tEnd) {
+ throw e;
+ }
+ }
+ Thread.sleep(1);
+ }
+ }
+
+ private void send(CacheInvalidations invalidations) throws Exception {
+ var urlConn = CACHE_INVALIDATIONS_ENDPOINT.toURL().openConnection();
+ urlConn.setDoOutput(true);
+ urlConn.setRequestProperty("Content-Type", APPLICATION_JSON);
+ urlConn.setRequestProperty("Polaris-Cache-Invalidation-Token", TOKEN);
+ try (var out = urlConn.getOutputStream()) {
+ out.write(mapper.writeValueAsBytes(invalidations));
+ }
+ urlConn.getInputStream().readAllBytes();
+ }
+}
