This is an automated email from the ASF dual-hosted git repository.
dimas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 4ffb98af1 feat(metrics): Add schema-v4 with metrics tables support
(#3523)
4ffb98af1 is described below
commit 4ffb98af1a8aeaea8a1647e62c2836b6541eb655
Author: Anand K Sankaran <[email protected]>
AuthorDate: Tue Feb 17 12:08:40 2026 -0800
feat(metrics): Add schema-v4 with metrics tables support (#3523)
This commit adds database schema support for metrics persistence tables
as part of the JDBC persistence backend.
Key changes:
- schema-metrics-v1.sql for PostgreSQL and H2 with metrics tables
- DatabaseType: Add metrics schema resource path support
- JdbcBasePersistenceImpl: Add metrics schema bootstrap capability
- JdbcBootstrapUtils: Support for metrics schema initialization
- QueryGenerator: Add metrics-related query generation
- SchemaOptions: Add metrics schema option for bootstrap command
- BootstrapCommand: Support --metrics flag for schema bootstrap
Testing:
- MetricsPersistenceBootstrapValidationTest: Validates schema bootstrap
- JdbcBootstrapUtilsTest: Tests for bootstrap utilities
- QueryGeneratorTest: Tests for metrics query generation
Co-authored-by: Anand Kumar Sankaran <[email protected]>
---
.../persistence/relational/jdbc/DatabaseType.java | 20 +++
.../relational/jdbc/DatasourceOperations.java | 8 +-
.../relational/jdbc/JdbcBasePersistenceImpl.java | 19 +++
.../relational/jdbc/JdbcBootstrapUtils.java | 14 ++
.../jdbc/JdbcMetaStoreManagerFactory.java | 7 +
.../relational/jdbc/QueryGenerator.java | 7 +
.../src/main/resources/h2/schema-metrics-v1.sql | 166 +++++++++++++++++++
.../main/resources/postgres/schema-metrics-v1.sql | 177 +++++++++++++++++++++
.../relational/jdbc/JdbcBootstrapUtilsTest.java | 31 ++++
.../MetricsPersistenceBootstrapValidationTest.java | 159 ++++++++++++++++++
.../relational/jdbc/QueryGeneratorTest.java | 10 ++
.../core/persistence/bootstrap/SchemaOptions.java | 7 +
.../apache/polaris/admintool/BootstrapCommand.java | 9 ++
.../jdbc/RelationalJdbcBootstrapCommandTest.java | 23 +++
14 files changed, 655 insertions(+), 2 deletions(-)
diff --git
a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java
b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java
index 71a33db21..28b77ad8c 100644
---
a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java
+++
b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java
@@ -60,4 +60,24 @@ public enum DatabaseType {
ClassLoader classLoader = DatasourceOperations.class.getClassLoader();
return classLoader.getResourceAsStream(resourceName);
}
+
+ /**
+ * Open an InputStream that contains data from the metrics schema init
script. This stream should
+ * be closed by the caller.
+ *
+ * @param metricsSchemaVersion the metrics schema version (currently only 1
is supported)
+ * @return an InputStream for the metrics schema SQL file
+ */
+ public InputStream openMetricsSchemaResource(int metricsSchemaVersion) {
+ if (metricsSchemaVersion != 1) {
+ throw new IllegalArgumentException(
+ "Unknown or invalid metrics schema version " + metricsSchemaVersion);
+ }
+
+ final String resourceName =
+ String.format("%s/schema-metrics-v%d.sql", this.getDisplayName(),
metricsSchemaVersion);
+
+ ClassLoader classLoader = DatasourceOperations.class.getClassLoader();
+ return classLoader.getResourceAsStream(resourceName);
+ }
}
diff --git
a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java
b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java
index e44de3a94..5e1b2785d 100644
---
a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java
+++
b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java
@@ -57,7 +57,9 @@ public class DatasourceOperations {
private static final String RELATION_DOES_NOT_EXIST = "42P01";
// H2 STATUS CODES
- private static final String H2_RELATION_DOES_NOT_EXIST = "90079";
+ // 90079 = Schema not found, 42S02 = Table or view not found
+ private static final String H2_SCHEMA_DOES_NOT_EXIST = "90079";
+ private static final String H2_TABLE_DOES_NOT_EXIST = "42S02";
// POSTGRES RETRYABLE EXCEPTIONS
private static final String SERIALIZATION_FAILURE_SQL_CODE = "40001";
@@ -402,7 +404,9 @@ public class DatasourceOperations {
public boolean isRelationDoesNotExist(SQLException e) {
return (RELATION_DOES_NOT_EXIST.equals(e.getSQLState())
&& databaseType == DatabaseType.POSTGRES)
- || (H2_RELATION_DOES_NOT_EXIST.equals(e.getSQLState()) && databaseType
== DatabaseType.H2);
+ || ((H2_SCHEMA_DOES_NOT_EXIST.equals(e.getSQLState())
+ || H2_TABLE_DOES_NOT_EXIST.equals(e.getSQLState()))
+ && databaseType == DatabaseType.H2);
}
private Connection borrowConnection() throws SQLException {
diff --git
a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java
b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java
index 9401df2dd..f3a83cbf0 100644
---
a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java
+++
b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java
@@ -777,6 +777,25 @@ public class JdbcBasePersistenceImpl implements
BasePersistence, IntegrationPers
}
}
+ /**
+ * Checks if the metrics tables have been bootstrapped by querying the
metrics_version table.
+ *
+ * @param datasourceOperations the datasource operations to use for the check
+ * @return true if the metrics_version table exists and contains data, false
otherwise
+ */
+ public static boolean metricsTableExists(DatasourceOperations
datasourceOperations) {
+ PreparedQuery query = QueryGenerator.generateMetricsVersionQuery();
+ try {
+ List<SchemaVersion> versions = datasourceOperations.executeSelect(query,
new SchemaVersion());
+ return versions != null && !versions.isEmpty();
+ } catch (SQLException e) {
+ if (datasourceOperations.isRelationDoesNotExist(e)) {
+ return false;
+ }
+ throw new IllegalStateException("Failed to check if metrics tables
exist", e);
+ }
+ }
+
/** {@inheritDoc} */
@Override
public <T extends PolarisEntity & LocationBasedEntity>
diff --git
a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtils.java
b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtils.java
index 814417d1b..a6e691cd0 100644
---
a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtils.java
+++
b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtils.java
@@ -88,4 +88,18 @@ public class JdbcBootstrapUtils {
}
return -1;
}
+
+ /**
+ * Determines whether the metrics schema should be included during bootstrap.
+ *
+ * @param bootstrapOptions The bootstrap options containing schema
information.
+ * @return true if the metrics schema should be included, false otherwise.
+ */
+ public static boolean shouldIncludeMetrics(BootstrapOptions
bootstrapOptions) {
+ SchemaOptions schemaOptions = bootstrapOptions.schemaOptions();
+ if (schemaOptions != null) {
+ return schemaOptions.includeMetrics();
+ }
+ return false;
+ }
}
diff --git
a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java
b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java
index 26f38fc31..24c1d025c 100644
---
a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java
+++
b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java
@@ -172,6 +172,13 @@ public class JdbcMetaStoreManagerFactory implements
MetaStoreManagerFactory {
datasourceOperations
.getDatabaseType()
.openInitScriptResource(effectiveSchemaVersion));
+
+ // Run the metrics schema script if requested
+ if (JdbcBootstrapUtils.shouldIncludeMetrics(bootstrapOptions)) {
+ LOGGER.info("Including metrics schema for realm: {}", realm);
+ datasourceOperations.executeScript(
+
datasourceOperations.getDatabaseType().openMetricsSchemaResource(1));
+ }
} catch (SQLException e) {
throw new RuntimeException(
String.format("Error executing sql script: %s", e.getMessage()),
e);
diff --git
a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java
b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java
index a8720fda5..044625276 100644
---
a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java
+++
b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java
@@ -369,6 +369,13 @@ public class QueryGenerator {
List.of());
}
+ @VisibleForTesting
+ static PreparedQuery generateMetricsVersionQuery() {
+ return new PreparedQuery(
+ "SELECT version_value FROM POLARIS_SCHEMA.metrics_version WHERE
version_key = 'metrics_version'",
+ List.of());
+ }
+
/**
* Generate a SELECT query to find any entities that have a given realm
& parent and that may
* overlap with a given location. The check is performed without
consideration for the scheme, so
diff --git
a/persistence/relational-jdbc/src/main/resources/h2/schema-metrics-v1.sql
b/persistence/relational-jdbc/src/main/resources/h2/schema-metrics-v1.sql
new file mode 100644
index 000000000..9f73a567b
--- /dev/null
+++ b/persistence/relational-jdbc/src/main/resources/h2/schema-metrics-v1.sql
@@ -0,0 +1,166 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file--
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"). You may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+
+-- This schema is SEPARATE from the entity schema and can evolve independently.
+-- It contains tables for storing Iceberg metrics reports.
+--
+-- Tables:
+-- * `metrics_version` - Version tracking for the metrics schema
+-- * `scan_metrics_report` - Scan metrics reports
+-- * `commit_metrics_report` - Commit metrics reports
+-- ============================================================================
+
+CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA;
+SET SCHEMA POLARIS_SCHEMA;
+
+-- Metrics schema version tracking (separate from entity schema version)
+CREATE TABLE IF NOT EXISTS metrics_version (
+ version_key VARCHAR PRIMARY KEY,
+ version_value INTEGER NOT NULL
+);
+
+MERGE INTO metrics_version (version_key, version_value)
+ KEY (version_key)
+ VALUES ('metrics_version', 1);
+
+COMMENT ON TABLE metrics_version IS 'the version of the metrics schema in use';
+
+-- ============================================================================
+-- SCAN METRICS REPORT TABLE
+-- ============================================================================
+
+CREATE TABLE IF NOT EXISTS scan_metrics_report (
+ report_id TEXT NOT NULL,
+ realm_id TEXT NOT NULL,
+ catalog_id BIGINT NOT NULL,
+ table_id BIGINT NOT NULL,
+
+ -- Report metadata
+ timestamp_ms BIGINT NOT NULL,
+ principal_name TEXT,
+ request_id TEXT,
+
+ -- Trace correlation
+ otel_trace_id TEXT,
+ otel_span_id TEXT,
+ report_trace_id TEXT,
+
+ -- Scan context
+ snapshot_id BIGINT,
+ schema_id INTEGER,
+ filter_expression TEXT,
+ projected_field_ids TEXT,
+ projected_field_names TEXT,
+
+ -- Scan metrics
+ result_data_files BIGINT DEFAULT 0,
+ result_delete_files BIGINT DEFAULT 0,
+ total_file_size_bytes BIGINT DEFAULT 0,
+ total_data_manifests BIGINT DEFAULT 0,
+ total_delete_manifests BIGINT DEFAULT 0,
+ scanned_data_manifests BIGINT DEFAULT 0,
+ scanned_delete_manifests BIGINT DEFAULT 0,
+ skipped_data_manifests BIGINT DEFAULT 0,
+ skipped_delete_manifests BIGINT DEFAULT 0,
+ skipped_data_files BIGINT DEFAULT 0,
+ skipped_delete_files BIGINT DEFAULT 0,
+ total_planning_duration_ms BIGINT DEFAULT 0,
+
+ -- Equality/positional delete metrics
+ equality_delete_files BIGINT DEFAULT 0,
+ positional_delete_files BIGINT DEFAULT 0,
+ indexed_delete_files BIGINT DEFAULT 0,
+ total_delete_file_size_bytes BIGINT DEFAULT 0,
+
+ -- Additional metadata (for extensibility)
+ metadata TEXT DEFAULT '{}',
+
+ PRIMARY KEY (realm_id, report_id)
+);
+
+COMMENT ON TABLE scan_metrics_report IS 'Scan metrics reports as first-class
entities';
+
+-- Index for retention cleanup by timestamp
+CREATE INDEX IF NOT EXISTS idx_scan_report_timestamp ON
scan_metrics_report(realm_id, timestamp_ms);
+
+-- ============================================================================
+-- COMMIT METRICS REPORT TABLE
+-- ============================================================================
+
+CREATE TABLE IF NOT EXISTS commit_metrics_report (
+ report_id TEXT NOT NULL,
+ realm_id TEXT NOT NULL,
+ catalog_id BIGINT NOT NULL,
+ table_id BIGINT NOT NULL,
+
+ -- Report metadata
+ timestamp_ms BIGINT NOT NULL,
+ principal_name TEXT,
+ request_id TEXT,
+
+ -- Trace correlation
+ otel_trace_id TEXT,
+ otel_span_id TEXT,
+ report_trace_id TEXT,
+
+ -- Commit context
+ snapshot_id BIGINT NOT NULL,
+ sequence_number BIGINT,
+ operation TEXT NOT NULL,
+
+ -- File metrics
+ added_data_files BIGINT DEFAULT 0,
+ removed_data_files BIGINT DEFAULT 0,
+ total_data_files BIGINT DEFAULT 0,
+ added_delete_files BIGINT DEFAULT 0,
+ removed_delete_files BIGINT DEFAULT 0,
+ total_delete_files BIGINT DEFAULT 0,
+
+ -- Equality delete files
+ added_equality_delete_files BIGINT DEFAULT 0,
+ removed_equality_delete_files BIGINT DEFAULT 0,
+
+ -- Positional delete files
+ added_positional_delete_files BIGINT DEFAULT 0,
+ removed_positional_delete_files BIGINT DEFAULT 0,
+
+ -- Record metrics
+ added_records BIGINT DEFAULT 0,
+ removed_records BIGINT DEFAULT 0,
+ total_records BIGINT DEFAULT 0,
+
+ -- Size metrics
+ added_file_size_bytes BIGINT DEFAULT 0,
+ removed_file_size_bytes BIGINT DEFAULT 0,
+ total_file_size_bytes BIGINT DEFAULT 0,
+
+ -- Duration and attempts
+ total_duration_ms BIGINT DEFAULT 0,
+ attempts INTEGER DEFAULT 1,
+
+ -- Additional metadata (for extensibility)
+ metadata TEXT DEFAULT '{}',
+
+ PRIMARY KEY (realm_id, report_id)
+);
+
+COMMENT ON TABLE commit_metrics_report IS 'Commit metrics reports as
first-class entities';
+
+-- Index for retention cleanup by timestamp
+CREATE INDEX IF NOT EXISTS idx_commit_report_timestamp ON
commit_metrics_report(realm_id, timestamp_ms);
diff --git
a/persistence/relational-jdbc/src/main/resources/postgres/schema-metrics-v1.sql
b/persistence/relational-jdbc/src/main/resources/postgres/schema-metrics-v1.sql
new file mode 100644
index 000000000..753f861cb
--- /dev/null
+++
b/persistence/relational-jdbc/src/main/resources/postgres/schema-metrics-v1.sql
@@ -0,0 +1,177 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file--
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"). You may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- This schema is SEPARATE from the entity schema and can evolve independently.
+-- It contains tables for storing Iceberg metrics reports.
+--
+-- Tables:
+-- * `metrics_version` - Version tracking for the metrics schema
+-- * `scan_metrics_report` - Scan metrics reports
+-- * `commit_metrics_report` - Commit metrics reports
+-- ============================================================================
+
+CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA;
+SET search_path TO POLARIS_SCHEMA;
+
+-- Metrics schema version tracking (separate from entity schema version)
+CREATE TABLE IF NOT EXISTS metrics_version (
+ version_key TEXT PRIMARY KEY,
+ version_value INTEGER NOT NULL
+);
+
+INSERT INTO metrics_version (version_key, version_value)
+VALUES ('metrics_version', 1)
+ON CONFLICT (version_key) DO UPDATE
+SET version_value = EXCLUDED.version_value;
+
+COMMENT ON TABLE metrics_version IS 'the version of the metrics schema in use';
+
+-- ============================================================================
+-- SCAN METRICS REPORT TABLE
+-- ============================================================================
+
+CREATE TABLE IF NOT EXISTS scan_metrics_report (
+ report_id TEXT NOT NULL,
+ realm_id TEXT NOT NULL,
+ catalog_id BIGINT NOT NULL,
+ table_id BIGINT NOT NULL,
+
+ -- Report metadata
+ timestamp_ms BIGINT NOT NULL,
+ principal_name TEXT,
+ request_id TEXT,
+
+ -- Trace correlation
+ otel_trace_id TEXT,
+ otel_span_id TEXT,
+ report_trace_id TEXT,
+
+ -- Scan context
+ snapshot_id BIGINT,
+ schema_id INTEGER,
+ filter_expression TEXT,
+ projected_field_ids TEXT,
+ projected_field_names TEXT,
+
+ -- Scan metrics
+ result_data_files BIGINT DEFAULT 0,
+ result_delete_files BIGINT DEFAULT 0,
+ total_file_size_bytes BIGINT DEFAULT 0,
+ total_data_manifests BIGINT DEFAULT 0,
+ total_delete_manifests BIGINT DEFAULT 0,
+ scanned_data_manifests BIGINT DEFAULT 0,
+ scanned_delete_manifests BIGINT DEFAULT 0,
+ skipped_data_manifests BIGINT DEFAULT 0,
+ skipped_delete_manifests BIGINT DEFAULT 0,
+ skipped_data_files BIGINT DEFAULT 0,
+ skipped_delete_files BIGINT DEFAULT 0,
+ total_planning_duration_ms BIGINT DEFAULT 0,
+
+ -- Equality/positional delete metrics
+ equality_delete_files BIGINT DEFAULT 0,
+ positional_delete_files BIGINT DEFAULT 0,
+ indexed_delete_files BIGINT DEFAULT 0,
+ total_delete_file_size_bytes BIGINT DEFAULT 0,
+
+ -- Additional metadata (for extensibility)
+ metadata JSONB DEFAULT '{}'::JSONB,
+
+ PRIMARY KEY (realm_id, report_id)
+);
+
+COMMENT ON TABLE scan_metrics_report IS 'Scan metrics reports as first-class
entities';
+COMMENT ON COLUMN scan_metrics_report.report_id IS 'Unique identifier for the
report';
+COMMENT ON COLUMN scan_metrics_report.realm_id IS 'Realm ID for multi-tenancy';
+COMMENT ON COLUMN scan_metrics_report.catalog_id IS 'Catalog ID';
+COMMENT ON COLUMN scan_metrics_report.otel_trace_id IS 'OpenTelemetry trace ID
from HTTP headers';
+COMMENT ON COLUMN scan_metrics_report.report_trace_id IS 'Trace ID from report
metadata';
+
+-- Index for retention cleanup by timestamp
+CREATE INDEX IF NOT EXISTS idx_scan_report_timestamp
+ ON scan_metrics_report(realm_id, timestamp_ms DESC);
+
+-- ============================================================================
+-- COMMIT METRICS REPORT TABLE
+-- ============================================================================
+
+CREATE TABLE IF NOT EXISTS commit_metrics_report (
+ report_id TEXT NOT NULL,
+ realm_id TEXT NOT NULL,
+ catalog_id BIGINT NOT NULL,
+ table_id BIGINT NOT NULL,
+
+ -- Report metadata
+ timestamp_ms BIGINT NOT NULL,
+ principal_name TEXT,
+ request_id TEXT,
+
+ -- Trace correlation
+ otel_trace_id TEXT,
+ otel_span_id TEXT,
+ report_trace_id TEXT,
+
+ -- Commit context
+ snapshot_id BIGINT NOT NULL,
+ sequence_number BIGINT,
+ operation TEXT NOT NULL,
+
+ -- File metrics
+ added_data_files BIGINT DEFAULT 0,
+ removed_data_files BIGINT DEFAULT 0,
+ total_data_files BIGINT DEFAULT 0,
+ added_delete_files BIGINT DEFAULT 0,
+ removed_delete_files BIGINT DEFAULT 0,
+ total_delete_files BIGINT DEFAULT 0,
+
+ -- Equality delete files
+ added_equality_delete_files BIGINT DEFAULT 0,
+ removed_equality_delete_files BIGINT DEFAULT 0,
+
+ -- Positional delete files
+ added_positional_delete_files BIGINT DEFAULT 0,
+ removed_positional_delete_files BIGINT DEFAULT 0,
+
+ -- Record metrics
+ added_records BIGINT DEFAULT 0,
+ removed_records BIGINT DEFAULT 0,
+ total_records BIGINT DEFAULT 0,
+
+ -- Size metrics
+ added_file_size_bytes BIGINT DEFAULT 0,
+ removed_file_size_bytes BIGINT DEFAULT 0,
+ total_file_size_bytes BIGINT DEFAULT 0,
+
+ -- Duration and attempts
+ total_duration_ms BIGINT DEFAULT 0,
+ attempts INTEGER DEFAULT 1,
+
+ -- Additional metadata (for extensibility)
+ metadata JSONB DEFAULT '{}'::JSONB,
+
+ PRIMARY KEY (realm_id, report_id)
+);
+
+COMMENT ON TABLE commit_metrics_report IS 'Commit metrics reports as
first-class entities';
+COMMENT ON COLUMN commit_metrics_report.report_id IS 'Unique identifier for
the report';
+COMMENT ON COLUMN commit_metrics_report.realm_id IS 'Realm ID for
multi-tenancy';
+COMMENT ON COLUMN commit_metrics_report.operation IS 'Commit operation type:
append, overwrite, delete, replace';
+COMMENT ON COLUMN commit_metrics_report.otel_trace_id IS 'OpenTelemetry trace
ID from HTTP headers';
+
+-- Index for retention cleanup by timestamp
+CREATE INDEX IF NOT EXISTS idx_commit_report_timestamp
+ ON commit_metrics_report(realm_id, timestamp_ms DESC);
diff --git
a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtilsTest.java
b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtilsTest.java
index 6a9eb9552..f36ab6848 100644
---
a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtilsTest.java
+++
b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtilsTest.java
@@ -150,4 +150,35 @@ class JdbcBootstrapUtilsTest {
assertEquals(-1, result);
}
}
+
+ @Nested
+ @ExtendWith(MockitoExtension.class)
+ class ShouldIncludeMetricsTests {
+
+ @Mock private BootstrapOptions mockBootstrapOptions;
+ @Mock private SchemaOptions mockSchemaOptions;
+
+ @Test
+ void whenSchemaOptionsIsNull_shouldReturnFalse() {
+ when(mockBootstrapOptions.schemaOptions()).thenReturn(null);
+ boolean result =
JdbcBootstrapUtils.shouldIncludeMetrics(mockBootstrapOptions);
+ assertEquals(false, result);
+ }
+
+ @Test
+ void whenIncludeMetricsIsTrue_shouldReturnTrue() {
+ when(mockBootstrapOptions.schemaOptions()).thenReturn(mockSchemaOptions);
+ when(mockSchemaOptions.includeMetrics()).thenReturn(true);
+ boolean result =
JdbcBootstrapUtils.shouldIncludeMetrics(mockBootstrapOptions);
+ assertEquals(true, result);
+ }
+
+ @Test
+ void whenIncludeMetricsIsFalse_shouldReturnFalse() {
+ when(mockBootstrapOptions.schemaOptions()).thenReturn(mockSchemaOptions);
+ when(mockSchemaOptions.includeMetrics()).thenReturn(false);
+ boolean result =
JdbcBootstrapUtils.shouldIncludeMetrics(mockBootstrapOptions);
+ assertEquals(false, result);
+ }
+ }
}
diff --git
a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/MetricsPersistenceBootstrapValidationTest.java
b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/MetricsPersistenceBootstrapValidationTest.java
new file mode 100644
index 000000000..f92f77f7f
--- /dev/null
+++
b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/MetricsPersistenceBootstrapValidationTest.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.persistence.relational.jdbc;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.InputStream;
+import java.sql.SQLException;
+import java.util.Optional;
+import javax.sql.DataSource;
+import org.h2.jdbcx.JdbcConnectionPool;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for metrics persistence bootstrap validation functionality.
+ *
+ * <p>These tests verify that the system correctly detects whether the metrics
tables have been
+ * bootstrapped and provides appropriate error messages when they are missing.
+ */
+class MetricsPersistenceBootstrapValidationTest {
+
+ private DataSource dataSource;
+ private DatasourceOperations datasourceOperations;
+
+ @BeforeEach
+ void setUp() throws SQLException {
+ // Create a fresh H2 in-memory database for each test
+ dataSource =
+ JdbcConnectionPool.create(
+ "jdbc:h2:mem:test_metrics_validation_" + System.nanoTime() +
";DB_CLOSE_DELAY=-1",
+ "sa",
+ "");
+ datasourceOperations = new DatasourceOperations(dataSource, new
TestJdbcConfiguration());
+ }
+
+ /** Test configuration for H2 database. */
+ private static class TestJdbcConfiguration implements
RelationalJdbcConfiguration {
+ @Override
+ public Optional<Integer> maxRetries() {
+ return Optional.of(2);
+ }
+
+ @Override
+ public Optional<Long> maxDurationInMs() {
+ return Optional.of(100L);
+ }
+
+ @Override
+ public Optional<Long> initialDelayInMs() {
+ return Optional.of(100L);
+ }
+ }
+
+ @AfterEach
+ void tearDown() {
+ if (dataSource instanceof JdbcConnectionPool) {
+ ((JdbcConnectionPool) dataSource).dispose();
+ }
+ }
+
+ @Nested
+ class MetricsTableExistsTests {
+
+ @Test
+ void whenMetricsTableDoesNotExist_shouldReturnFalse() {
+ // No schema loaded - metrics table doesn't exist
+ boolean result =
JdbcBasePersistenceImpl.metricsTableExists(datasourceOperations);
+ assertThat(result).isFalse();
+ }
+
+ @Test
+ void whenOnlyEntitySchemaLoaded_shouldReturnFalse() throws SQLException {
+ // Load only the entity schema (v4), not the metrics schema
+ loadSchema("h2/schema-v4.sql");
+
+ boolean result =
JdbcBasePersistenceImpl.metricsTableExists(datasourceOperations);
+ assertThat(result).isFalse();
+ }
+
+ @Test
+ void whenMetricsSchemaLoaded_shouldReturnTrue() throws SQLException {
+ // Load the metrics schema
+ loadSchema("h2/schema-metrics-v1.sql");
+
+ boolean result =
JdbcBasePersistenceImpl.metricsTableExists(datasourceOperations);
+ assertThat(result).isTrue();
+ }
+
+ @Test
+ void whenBothSchemasLoaded_shouldReturnTrue() throws SQLException {
+ // Load both entity and metrics schemas
+ loadSchema("h2/schema-v4.sql");
+ loadSchema("h2/schema-metrics-v1.sql");
+
+ boolean result =
JdbcBasePersistenceImpl.metricsTableExists(datasourceOperations);
+ assertThat(result).isTrue();
+ }
+ }
+
+ @Nested
+ class CheckMetricsPersistenceBootstrappedTests {
+
+ @Test
+ void whenMetricsTableDoesNotExist_shouldThrowIllegalStateException() {
+ // No schema loaded - metrics table doesn't exist
+ assertThatThrownBy(
+ () -> {
+ if
(!JdbcBasePersistenceImpl.metricsTableExists(datasourceOperations)) {
+ throw new IllegalStateException(
+ "Metrics persistence is enabled but the metrics tables
have not been bootstrapped. "
+ + "Please run the bootstrap command with the
--include-metrics flag to create "
+ + "the required schema before enabling this
feature.");
+ }
+ })
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("metrics tables have not been bootstrapped")
+ .hasMessageContaining("--include-metrics");
+ }
+
+ @Test
+ void whenMetricsSchemaLoaded_shouldNotThrow() throws SQLException {
+ // Load the metrics schema
+ loadSchema("h2/schema-metrics-v1.sql");
+
+ // Should not throw
+ boolean exists =
JdbcBasePersistenceImpl.metricsTableExists(datasourceOperations);
+ assertThat(exists).isTrue();
+ }
+ }
+
+ private void loadSchema(String schemaPath) throws SQLException {
+ ClassLoader classLoader = getClass().getClassLoader();
+ InputStream scriptStream = classLoader.getResourceAsStream(schemaPath);
+ if (scriptStream == null) {
+ throw new IllegalStateException("Schema file not found: " + schemaPath);
+ }
+ datasourceOperations.executeScript(scriptStream);
+ }
+}
diff --git
a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java
b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java
index 38a06f2b5..2f6d481d1 100644
---
a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java
+++
b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java
@@ -33,6 +33,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.polaris.core.entity.PolarisEntityCore;
import org.apache.polaris.core.entity.PolarisEntityId;
+import
org.apache.polaris.persistence.relational.jdbc.QueryGenerator.PreparedQuery;
import org.apache.polaris.persistence.relational.jdbc.models.ModelEntity;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -395,4 +396,13 @@ public class QueryGeneratorTest {
.containsExactly(
"realmId", -123L, "/", "//", "//バケツ/", "//バケツ/\"loc.ation\"/",
"//バケツ/\"loc.ation\"/%");
}
+
+ @Test
+ void testGenerateMetricsVersionQuery() {
+ PreparedQuery query = QueryGenerator.generateMetricsVersionQuery();
+ assertEquals(
+ "SELECT version_value FROM POLARIS_SCHEMA.metrics_version WHERE
version_key = 'metrics_version'",
+ query.sql());
+ Assertions.assertThat(query.parameters()).isEmpty();
+ }
}
diff --git
a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java
b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java
index 5cfc20a88..8798a66f9 100644
---
a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java
+++
b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java
@@ -21,8 +21,15 @@ package org.apache.polaris.core.persistence.bootstrap;
import java.util.Optional;
import org.apache.polaris.immutables.PolarisImmutable;
+import org.immutables.value.Value;
@PolarisImmutable
public interface SchemaOptions {
Optional<Integer> schemaVersion();
+
+ /** Whether to include the metrics schema during bootstrap. Defaults to
false. */
+ @Value.Default
+ default boolean includeMetrics() {
+ return false;
+ }
}
diff --git
a/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java
b/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java
index 82d92f4e1..53c89ddd5 100644
---
a/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java
+++
b/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java
@@ -93,6 +93,11 @@ public class BootstrapCommand extends BaseMetaStoreCommand {
paramLabel = "<schema version>",
description = "The version of the schema to load in [1, 2, 3,
LATEST].")
Integer schemaVersion;
+
+ @CommandLine.Option(
+ names = {"--include-metrics"},
+ description = "Include metrics schema tables during bootstrap.")
+ boolean includeMetrics;
}
}
@@ -136,6 +141,10 @@ public class BootstrapCommand extends BaseMetaStoreCommand
{
builder.schemaVersion(inputOptions.schemaInputOptions.schemaVersion);
}
+ if (inputOptions.schemaInputOptions.includeMetrics) {
+ builder.includeMetrics(true);
+ }
+
schemaOptions = builder.build();
} else {
schemaOptions = ImmutableSchemaOptions.builder().build();
diff --git
a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java
b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java
index 31f3a9eea..73abd2cbd 100644
---
a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java
+++
b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java
@@ -44,4 +44,27 @@ public class RelationalJdbcBootstrapCommandTest extends
BootstrapCommandTestBase
// assertThat(result2.exitCode()).isEqualTo(EXIT_CODE_BOOTSTRAP_ERROR);
// assertThat(result2.getOutput()).contains("Cannot bootstrap due to
schema version mismatch.");
}
+
+ @Test
+ public void testBootstrapWithIncludeMetrics(QuarkusMainLauncher launcher) {
+ // Test that --include-metrics option is accepted and bootstrap completes
successfully.
+ // The metrics tables are created during bootstrap when this flag is set.
+ LaunchResult result =
+ launcher.launch(
+ "bootstrap", "-r", "realm1", "-c", "realm1,root,s3cr3t",
"--include-metrics");
+ assertThat(result.exitCode()).isEqualTo(0);
+ assertThat(result.getOutput())
+ .contains("Realm 'realm1' successfully bootstrapped.")
+ .contains("Bootstrap completed successfully.");
+ }
+
+ @Test
+ public void testBootstrapWithoutIncludeMetrics(QuarkusMainLauncher launcher)
{
+ // Test that bootstrap works without --include-metrics (default behavior)
+ LaunchResult result = launcher.launch("bootstrap", "-r", "realm1", "-c",
"realm1,root,s3cr3t");
+ assertThat(result.exitCode()).isEqualTo(0);
+ assertThat(result.getOutput())
+ .contains("Realm 'realm1' successfully bootstrapped.")
+ .contains("Bootstrap completed successfully.");
+ }
}
