singhpk234 commented on code in PR #3385: URL: https://github.com/apache/polaris/pull/3385#discussion_r2723174844
########## persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReport.java: ########## @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.models; + +import jakarta.annotation.Nullable; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.polaris.immutables.PolarisImmutable; +import org.apache.polaris.persistence.relational.jdbc.DatabaseType; +import org.immutables.value.Value; + +/** Model class for commit_metrics_report table - stores commit metrics as first-class entities. */ +@PolarisImmutable +public interface ModelCommitMetricsReport extends Converter<ModelCommitMetricsReport> { + String TABLE_NAME = "COMMIT_METRICS_REPORT"; + + // Column names + String REPORT_ID = "report_id"; + String REALM_ID = "realm_id"; + String CATALOG_ID = "catalog_id"; + String CATALOG_NAME = "catalog_name"; + String NAMESPACE = "namespace"; + String TABLE_NAME_COL = "table_name"; + String TIMESTAMP_MS = "timestamp_ms"; + String PRINCIPAL_NAME = "principal_name"; + String REQUEST_ID = "request_id"; + String OTEL_TRACE_ID = "otel_trace_id"; + String OTEL_SPAN_ID = "otel_span_id"; + String REPORT_TRACE_ID = "report_trace_id"; + String SNAPSHOT_ID = "snapshot_id"; + String SEQUENCE_NUMBER = "sequence_number"; + String OPERATION = "operation"; + String ADDED_DATA_FILES = "added_data_files"; + String REMOVED_DATA_FILES = "removed_data_files"; + String TOTAL_DATA_FILES = "total_data_files"; + String ADDED_DELETE_FILES = "added_delete_files"; + String REMOVED_DELETE_FILES = "removed_delete_files"; + String TOTAL_DELETE_FILES = "total_delete_files"; + String ADDED_EQUALITY_DELETE_FILES = "added_equality_delete_files"; + String REMOVED_EQUALITY_DELETE_FILES = "removed_equality_delete_files"; + String ADDED_POSITIONAL_DELETE_FILES = "added_positional_delete_files"; + String REMOVED_POSITIONAL_DELETE_FILES = "removed_positional_delete_files"; + String ADDED_RECORDS = "added_records"; + String REMOVED_RECORDS = "removed_records"; + String TOTAL_RECORDS = "total_records"; + String ADDED_FILE_SIZE_BYTES = "added_file_size_bytes"; + String REMOVED_FILE_SIZE_BYTES = "removed_file_size_bytes"; + String TOTAL_FILE_SIZE_BYTES = "total_file_size_bytes"; + String TOTAL_DURATION_MS = "total_duration_ms"; + String ATTEMPTS = "attempts"; + String METADATA = "metadata"; + + List<String> ALL_COLUMNS = + List.of( + REPORT_ID, + REALM_ID, + CATALOG_ID, + CATALOG_NAME, + NAMESPACE, + TABLE_NAME_COL, + TIMESTAMP_MS, + PRINCIPAL_NAME, + REQUEST_ID, + OTEL_TRACE_ID, + OTEL_SPAN_ID, + REPORT_TRACE_ID, + SNAPSHOT_ID, + SEQUENCE_NUMBER, + OPERATION, + ADDED_DATA_FILES, + REMOVED_DATA_FILES, + TOTAL_DATA_FILES, + ADDED_DELETE_FILES, + REMOVED_DELETE_FILES, + TOTAL_DELETE_FILES, + ADDED_EQUALITY_DELETE_FILES, + REMOVED_EQUALITY_DELETE_FILES, + ADDED_POSITIONAL_DELETE_FILES, + REMOVED_POSITIONAL_DELETE_FILES, + ADDED_RECORDS, + REMOVED_RECORDS, + TOTAL_RECORDS, + ADDED_FILE_SIZE_BYTES, + REMOVED_FILE_SIZE_BYTES, + TOTAL_FILE_SIZE_BYTES, + TOTAL_DURATION_MS, + ATTEMPTS, + METADATA); + + // Getters + String getReportId(); + + String getRealmId(); + + String getCatalogId(); + + String getCatalogName(); + + String getNamespace(); + + String getTableName(); + + long getTimestampMs(); + + @Nullable + String getPrincipalName(); + + @Nullable + String getRequestId(); + + @Nullable + String getOtelTraceId(); + + @Nullable + String getOtelSpanId(); + + @Nullable + String getReportTraceId(); + + long getSnapshotId(); + + @Nullable + Long getSequenceNumber(); + + String getOperation(); + + long getAddedDataFiles(); + + long getRemovedDataFiles(); + + long getTotalDataFiles(); + + long getAddedDeleteFiles(); + + long getRemovedDeleteFiles(); + + long getTotalDeleteFiles(); + + long getAddedEqualityDeleteFiles(); + + long getRemovedEqualityDeleteFiles(); + + long getAddedPositionalDeleteFiles(); + + long getRemovedPositionalDeleteFiles(); + + long getAddedRecords(); + + long getRemovedRecords(); + + long getTotalRecords(); + + long getAddedFileSizeBytes(); + + long getRemovedFileSizeBytes(); + + long getTotalFileSizeBytes(); + + long getTotalDurationMs(); + + int getAttempts(); + + @Nullable + String getMetadata(); + + /** + * Returns the activated principal roles associated with this report. This is populated from the + * junction table commit_metrics_report_roles. + */ + @Value.Default + default Set<String> getRoles() { + return Set.of(); + } + + @Override + default ModelCommitMetricsReport fromResultSet(ResultSet rs) throws SQLException { + return ImmutableModelCommitMetricsReport.builder() + .reportId(rs.getString(REPORT_ID)) + .realmId(rs.getString(REALM_ID)) + .catalogId(rs.getString(CATALOG_ID)) + .catalogName(rs.getString(CATALOG_NAME)) + .namespace(rs.getString(NAMESPACE)) + .tableName(rs.getString(TABLE_NAME_COL)) + .timestampMs(rs.getLong(TIMESTAMP_MS)) + .principalName(rs.getString(PRINCIPAL_NAME)) + .requestId(rs.getString(REQUEST_ID)) + .otelTraceId(rs.getString(OTEL_TRACE_ID)) + .otelSpanId(rs.getString(OTEL_SPAN_ID)) + .reportTraceId(rs.getString(REPORT_TRACE_ID)) + .snapshotId(rs.getLong(SNAPSHOT_ID)) + .sequenceNumber(rs.getObject(SEQUENCE_NUMBER, Long.class)) + .operation(rs.getString(OPERATION)) + .addedDataFiles(rs.getLong(ADDED_DATA_FILES)) + .removedDataFiles(rs.getLong(REMOVED_DATA_FILES)) + .totalDataFiles(rs.getLong(TOTAL_DATA_FILES)) + .addedDeleteFiles(rs.getLong(ADDED_DELETE_FILES)) + .removedDeleteFiles(rs.getLong(REMOVED_DELETE_FILES)) + .totalDeleteFiles(rs.getLong(TOTAL_DELETE_FILES)) + .addedEqualityDeleteFiles(rs.getLong(ADDED_EQUALITY_DELETE_FILES)) + .removedEqualityDeleteFiles(rs.getLong(REMOVED_EQUALITY_DELETE_FILES)) + .addedPositionalDeleteFiles(rs.getLong(ADDED_POSITIONAL_DELETE_FILES)) + .removedPositionalDeleteFiles(rs.getLong(REMOVED_POSITIONAL_DELETE_FILES)) + .addedRecords(rs.getLong(ADDED_RECORDS)) + .removedRecords(rs.getLong(REMOVED_RECORDS)) + .totalRecords(rs.getLong(TOTAL_RECORDS)) + .addedFileSizeBytes(rs.getLong(ADDED_FILE_SIZE_BYTES)) + .removedFileSizeBytes(rs.getLong(REMOVED_FILE_SIZE_BYTES)) + .totalFileSizeBytes(rs.getLong(TOTAL_FILE_SIZE_BYTES)) + .totalDurationMs(rs.getLong(TOTAL_DURATION_MS)) + .attempts(rs.getInt(ATTEMPTS)) + .metadata(rs.getString(METADATA)) + .build(); + } + + @Override + default Map<String, Object> toMap(DatabaseType databaseType) { + Map<String, Object> map = new LinkedHashMap<>(); + map.put(REPORT_ID, getReportId()); + map.put(REALM_ID, getRealmId()); + map.put(CATALOG_ID, getCatalogId()); + map.put(CATALOG_NAME, getCatalogName()); + map.put(NAMESPACE, getNamespace()); + map.put(TABLE_NAME_COL, getTableName()); + map.put(TIMESTAMP_MS, getTimestampMs()); + map.put(PRINCIPAL_NAME, getPrincipalName()); + map.put(REQUEST_ID, getRequestId()); + map.put(OTEL_TRACE_ID, getOtelTraceId()); + map.put(OTEL_SPAN_ID, getOtelSpanId()); + map.put(REPORT_TRACE_ID, getReportTraceId()); + map.put(SNAPSHOT_ID, getSnapshotId()); + map.put(SEQUENCE_NUMBER, getSequenceNumber()); + map.put(OPERATION, getOperation()); + map.put(ADDED_DATA_FILES, getAddedDataFiles()); + map.put(REMOVED_DATA_FILES, getRemovedDataFiles()); + map.put(TOTAL_DATA_FILES, getTotalDataFiles()); + map.put(ADDED_DELETE_FILES, getAddedDeleteFiles()); + map.put(REMOVED_DELETE_FILES, getRemovedDeleteFiles()); + map.put(TOTAL_DELETE_FILES, getTotalDeleteFiles()); + map.put(ADDED_EQUALITY_DELETE_FILES, getAddedEqualityDeleteFiles()); + map.put(REMOVED_EQUALITY_DELETE_FILES, getRemovedEqualityDeleteFiles()); + map.put(ADDED_POSITIONAL_DELETE_FILES, getAddedPositionalDeleteFiles()); + map.put(REMOVED_POSITIONAL_DELETE_FILES, getRemovedPositionalDeleteFiles()); + map.put(ADDED_RECORDS, getAddedRecords()); + map.put(REMOVED_RECORDS, getRemovedRecords()); + map.put(TOTAL_RECORDS, getTotalRecords()); + map.put(ADDED_FILE_SIZE_BYTES, getAddedFileSizeBytes()); + map.put(REMOVED_FILE_SIZE_BYTES, getRemovedFileSizeBytes()); + map.put(TOTAL_FILE_SIZE_BYTES, getTotalFileSizeBytes()); + map.put(TOTAL_DURATION_MS, getTotalDurationMs()); + map.put(ATTEMPTS, getAttempts()); + if (databaseType.equals(DatabaseType.POSTGRES)) { + map.put(METADATA, toJsonbPGobject(getMetadata() != null ? getMetadata() : "{}")); + } else { + map.put(METADATA, getMetadata() != null ? getMetadata() : "{}"); + } Review Comment: sorry i know this i bad will put an adapter change soo ########## persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReport.java: ########## @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.models; + +import jakarta.annotation.Nullable; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.polaris.immutables.PolarisImmutable; +import org.apache.polaris.persistence.relational.jdbc.DatabaseType; +import org.immutables.value.Value; + +/** Model class for scan_metrics_report table - stores scan metrics as first-class entities. */ +@PolarisImmutable +public interface ModelScanMetricsReport extends Converter<ModelScanMetricsReport> { + String TABLE_NAME = "SCAN_METRICS_REPORT"; + + // Column names + String REPORT_ID = "report_id"; + String REALM_ID = "realm_id"; + String CATALOG_ID = "catalog_id"; + String CATALOG_NAME = "catalog_name"; + String NAMESPACE = "namespace"; + String TABLE_NAME_COL = "table_name"; + String TIMESTAMP_MS = "timestamp_ms"; + String PRINCIPAL_NAME = "principal_name"; + String REQUEST_ID = "request_id"; + String OTEL_TRACE_ID = "otel_trace_id"; + String OTEL_SPAN_ID = "otel_span_id"; + String REPORT_TRACE_ID = "report_trace_id"; + String SNAPSHOT_ID = "snapshot_id"; Review Comment: do you wanna may be have some thing like BaseReportModel to get these key in a common class this would reduce the builders ########## persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/MetricsReportConverter.java: ########## @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.models; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; +import org.apache.iceberg.metrics.CommitMetricsResult; +import org.apache.iceberg.metrics.CommitReport; +import org.apache.iceberg.metrics.CounterResult; +import org.apache.iceberg.metrics.ScanMetricsResult; +import org.apache.iceberg.metrics.ScanReport; +import org.apache.iceberg.metrics.TimerResult; + +/** + * Converter utility class for transforming Iceberg metrics reports into persistence model classes. + */ +public final class MetricsReportConverter { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private MetricsReportConverter() { + // Utility class + } + + /** + * Converts an Iceberg ScanReport to a ModelScanMetricsReport. + * + * @param scanReport the Iceberg scan report + * @param realmId the realm ID for multi-tenancy + * @param catalogId the catalog ID + * @param catalogName the catalog name + * @param namespace the namespace (dot-separated) + * @param principalName the principal who initiated the scan (optional) + * @param requestId the request ID (optional) + * @param otelTraceId OpenTelemetry trace ID (optional) + * @param otelSpanId OpenTelemetry span ID (optional) + * @return the converted ModelScanMetricsReport + */ + public static ModelScanMetricsReport fromScanReport( + ScanReport scanReport, + String realmId, + String catalogId, + String catalogName, + String namespace, + @Nullable String principalName, + @Nullable String requestId, + @Nullable String otelTraceId, + @Nullable String otelSpanId) { + + String reportId = UUID.randomUUID().toString(); + long timestampMs = System.currentTimeMillis(); + + ScanMetricsResult metrics = scanReport.scanMetrics(); + + ImmutableModelScanMetricsReport.Builder builder = + ImmutableModelScanMetricsReport.builder() + .reportId(reportId) + .realmId(realmId) + .catalogId(catalogId) + .catalogName(catalogName) + .namespace(namespace) + .tableName(scanReport.tableName()) + .timestampMs(timestampMs) + .principalName(principalName) + .requestId(requestId) + .otelTraceId(otelTraceId) + .otelSpanId(otelSpanId) + .snapshotId(scanReport.snapshotId()) + .schemaId(scanReport.schemaId()) + .filterExpression(scanReport.filter() != null ? scanReport.filter().toString() : null) + .projectedFieldIds(formatIntegerList(scanReport.projectedFieldIds())) + .projectedFieldNames(formatStringList(scanReport.projectedFieldNames())); + + // Extract metrics values + if (metrics != null) { + builder + .resultDataFiles(getCounterValue(metrics.resultDataFiles())) + .resultDeleteFiles(getCounterValue(metrics.resultDeleteFiles())) + .totalFileSizeBytes(getCounterValue(metrics.totalFileSizeInBytes())) + .totalDataManifests(getCounterValue(metrics.totalDataManifests())) + .totalDeleteManifests(getCounterValue(metrics.totalDeleteManifests())) + .scannedDataManifests(getCounterValue(metrics.scannedDataManifests())) + .scannedDeleteManifests(getCounterValue(metrics.scannedDeleteManifests())) + .skippedDataManifests(getCounterValue(metrics.skippedDataManifests())) + .skippedDeleteManifests(getCounterValue(metrics.skippedDeleteManifests())) + .skippedDataFiles(getCounterValue(metrics.skippedDataFiles())) + .skippedDeleteFiles(getCounterValue(metrics.skippedDeleteFiles())) + .totalPlanningDurationMs(getTimerValueMs(metrics.totalPlanningDuration())) + .equalityDeleteFiles(getCounterValue(metrics.equalityDeleteFiles())) + .positionalDeleteFiles(getCounterValue(metrics.positionalDeleteFiles())) + .indexedDeleteFiles(getCounterValue(metrics.indexedDeleteFiles())) + .totalDeleteFileSizeBytes(getCounterValue(metrics.totalDeleteFileSizeInBytes())); + } else { + builder + .resultDataFiles(0L) + .resultDeleteFiles(0L) + .totalFileSizeBytes(0L) + .totalDataManifests(0L) + .totalDeleteManifests(0L) + .scannedDataManifests(0L) + .scannedDeleteManifests(0L) + .skippedDataManifests(0L) + .skippedDeleteManifests(0L) + .skippedDataFiles(0L) + .skippedDeleteFiles(0L) + .totalPlanningDurationMs(0L) + .equalityDeleteFiles(0L) + .positionalDeleteFiles(0L) + .indexedDeleteFiles(0L) + .totalDeleteFileSizeBytes(0L); + } + + // Store additional metadata as JSON + Map<String, String> metadata = scanReport.metadata(); + if (metadata != null && !metadata.isEmpty()) { + builder.metadata(toJson(metadata)); + } + + return builder.build(); + } + + /** + * Converts an Iceberg CommitReport to a ModelCommitMetricsReport. + * + * @param commitReport the Iceberg commit report + * @param realmId the realm ID for multi-tenancy + * @param catalogId the catalog ID + * @param catalogName the catalog name + * @param namespace the namespace (dot-separated) + * @param principalName the principal who initiated the commit (optional) + * @param requestId the request ID (optional) + * @param otelTraceId OpenTelemetry trace ID (optional) + * @param otelSpanId OpenTelemetry span ID (optional) + * @return the converted ModelCommitMetricsReport + */ + public static ModelCommitMetricsReport fromCommitReport( + CommitReport commitReport, + String realmId, + String catalogId, + String catalogName, + String namespace, + @Nullable String principalName, + @Nullable String requestId, + @Nullable String otelTraceId, + @Nullable String otelSpanId) { + + String reportId = UUID.randomUUID().toString(); + long timestampMs = System.currentTimeMillis(); + + CommitMetricsResult metrics = commitReport.commitMetrics(); Review Comment: i wonder if metrics is null we just create and empty commit metric so that we skip is null metric check below ? (seems a bit redundant to me) ########## persistence/relational-jdbc/src/main/resources/postgres/schema-v4.sql: ########## @@ -0,0 +1,350 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"). You may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +-- ============================================================================ +-- POLARIS JDBC SCHEMA VERSION 4 +-- ============================================================================ +-- This schema is SELF-CONTAINED and can be used for fresh installs. +-- Each schema version includes ALL tables, not just incremental changes. +-- +-- Changes from v3: +-- * Added `scan_metrics_report` table for scan metrics as first-class entities +-- * Added `scan_metrics_report_roles` junction table for principal roles +-- * Added `commit_metrics_report` table for commit metrics as first-class entities +-- * Added `commit_metrics_report_roles` junction table for principal roles +-- ============================================================================ + +CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA; +SET search_path TO POLARIS_SCHEMA; + +-- ============================================================================ +-- VERSION TABLE +-- ============================================================================ +CREATE TABLE IF NOT EXISTS version ( + version_key TEXT PRIMARY KEY, + version_value INTEGER NOT NULL +); +INSERT INTO version (version_key, version_value) +VALUES ('version', 4) +ON CONFLICT (version_key) DO UPDATE +SET version_value = EXCLUDED.version_value; +COMMENT ON TABLE version IS 'the version of the JDBC schema in use'; + +-- ============================================================================ +-- CORE TABLES (from v1) +-- ============================================================================ + +-- Entities table: stores all Polaris entities (catalogs, namespaces, tables, etc.) +CREATE TABLE IF NOT EXISTS entities ( + realm_id TEXT NOT NULL, + catalog_id BIGINT NOT NULL, + id BIGINT NOT NULL, + parent_id BIGINT NOT NULL, + name TEXT NOT NULL, + entity_version INT NOT NULL, + type_code INT NOT NULL, + sub_type_code INT NOT NULL, + create_timestamp BIGINT NOT NULL, + drop_timestamp BIGINT NOT NULL, + purge_timestamp BIGINT NOT NULL, + to_purge_timestamp BIGINT NOT NULL, + last_update_timestamp BIGINT NOT NULL, + properties JSONB not null default '{}'::JSONB, + internal_properties JSONB not null default '{}'::JSONB, + grant_records_version INT NOT NULL, + location_without_scheme TEXT, + PRIMARY KEY (realm_id, id), + CONSTRAINT constraint_name UNIQUE (realm_id, catalog_id, parent_id, type_code, name) +); + +CREATE INDEX IF NOT EXISTS idx_entities ON entities (realm_id, catalog_id, id); +CREATE INDEX IF NOT EXISTS idx_locations + ON entities USING btree (realm_id, parent_id, location_without_scheme) + WHERE location_without_scheme IS NOT NULL; + +COMMENT ON TABLE entities IS 'all the entities'; +COMMENT ON COLUMN entities.realm_id IS 'realm_id used for multi-tenancy'; +COMMENT ON COLUMN entities.catalog_id IS 'catalog id'; +COMMENT ON COLUMN entities.id IS 'entity id'; +COMMENT ON COLUMN entities.parent_id IS 'entity id of parent'; +COMMENT ON COLUMN entities.name IS 'entity name'; +COMMENT ON COLUMN entities.entity_version IS 'version of the entity'; +COMMENT ON COLUMN entities.type_code IS 'type code'; +COMMENT ON COLUMN entities.sub_type_code IS 'sub type of entity'; +COMMENT ON COLUMN entities.create_timestamp IS 'creation time of entity'; +COMMENT ON COLUMN entities.drop_timestamp IS 'time of drop of entity'; +COMMENT ON COLUMN entities.purge_timestamp IS 'time to start purging entity'; +COMMENT ON COLUMN entities.last_update_timestamp IS 'last time the entity is touched'; +COMMENT ON COLUMN entities.properties IS 'entities properties json'; +COMMENT ON COLUMN entities.internal_properties IS 'entities internal properties json'; +COMMENT ON COLUMN entities.grant_records_version IS 'the version of grant records change on the entity'; + +-- Grant records table: stores privilege grants +CREATE TABLE IF NOT EXISTS grant_records ( + realm_id TEXT NOT NULL, + securable_catalog_id BIGINT NOT NULL, + securable_id BIGINT NOT NULL, + grantee_catalog_id BIGINT NOT NULL, + grantee_id BIGINT NOT NULL, + privilege_code INTEGER, + PRIMARY KEY (realm_id, securable_catalog_id, securable_id, grantee_catalog_id, grantee_id, privilege_code) +); + +COMMENT ON TABLE grant_records IS 'grant records for entities'; +COMMENT ON COLUMN grant_records.securable_catalog_id IS 'catalog id of the securable'; +COMMENT ON COLUMN grant_records.securable_id IS 'entity id of the securable'; +COMMENT ON COLUMN grant_records.grantee_catalog_id IS 'catalog id of the grantee'; +COMMENT ON COLUMN grant_records.grantee_id IS 'id of the grantee'; +COMMENT ON COLUMN grant_records.privilege_code IS 'privilege code'; + +-- Principal authentication data table +CREATE TABLE IF NOT EXISTS principal_authentication_data ( + realm_id TEXT NOT NULL, + principal_id BIGINT NOT NULL, + principal_client_id VARCHAR(255) NOT NULL, + main_secret_hash VARCHAR(255) NOT NULL, + secondary_secret_hash VARCHAR(255) NOT NULL, + secret_salt VARCHAR(255) NOT NULL, + PRIMARY KEY (realm_id, principal_client_id) +); + +COMMENT ON TABLE principal_authentication_data IS 'authentication data for client'; + +-- Policy mapping record table (from v2) +CREATE TABLE IF NOT EXISTS policy_mapping_record ( + realm_id TEXT NOT NULL, + target_catalog_id BIGINT NOT NULL, + target_id BIGINT NOT NULL, + policy_type_code INTEGER NOT NULL, + policy_catalog_id BIGINT NOT NULL, + policy_id BIGINT NOT NULL, + parameters JSONB NOT NULL DEFAULT '{}'::JSONB, + PRIMARY KEY (realm_id, target_catalog_id, target_id, policy_type_code, policy_catalog_id, policy_id) +); + +CREATE INDEX IF NOT EXISTS idx_policy_mapping_record ON policy_mapping_record (realm_id, policy_type_code, policy_catalog_id, policy_id, target_catalog_id, target_id); + +-- ============================================================================ +-- EVENTS TABLE (from v3) +-- ============================================================================ + +CREATE TABLE IF NOT EXISTS events ( + realm_id TEXT NOT NULL, + catalog_id TEXT NOT NULL, + event_id TEXT NOT NULL, + request_id TEXT, + event_type TEXT NOT NULL, + timestamp_ms BIGINT NOT NULL, + principal_name TEXT, + resource_type TEXT NOT NULL, + resource_identifier TEXT NOT NULL, + additional_properties JSONB NOT NULL DEFAULT '{}'::JSONB, + PRIMARY KEY (event_id) +); + +-- Idempotency records (from v3) +CREATE TABLE IF NOT EXISTS idempotency_records ( + realm_id TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + operation_type TEXT NOT NULL, + resource_id TEXT NOT NULL, + + -- Finalization/replay + http_status INTEGER, + error_subtype TEXT, + response_summary TEXT, + response_headers TEXT, + finalized_at TIMESTAMP, + + -- Liveness/ops + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + heartbeat_at TIMESTAMP, + executor_id TEXT, + expires_at TIMESTAMP, + + PRIMARY KEY (realm_id, idempotency_key) +); + +CREATE INDEX IF NOT EXISTS idx_idemp_realm_expires + ON idempotency_records (realm_id, expires_at); + +-- ============================================================================ +-- METRICS TABLES (NEW in v4) +-- ============================================================================ + +-- Scan Metrics Report Table +CREATE TABLE IF NOT EXISTS scan_metrics_report ( + report_id TEXT NOT NULL, + realm_id TEXT NOT NULL, + catalog_id TEXT NOT NULL, + catalog_name TEXT NOT NULL, + namespace TEXT NOT NULL, + table_name TEXT NOT NULL, + + -- Report metadata + timestamp_ms BIGINT NOT NULL, + principal_name TEXT, + request_id TEXT, + + -- Trace correlation + otel_trace_id TEXT, + otel_span_id TEXT, + report_trace_id TEXT, + + -- Scan context + snapshot_id BIGINT, + schema_id INTEGER, + filter_expression TEXT, + projected_field_ids TEXT, + projected_field_names TEXT, + + -- Scan metrics + result_data_files BIGINT DEFAULT 0, + result_delete_files BIGINT DEFAULT 0, + total_file_size_bytes BIGINT DEFAULT 0, + total_data_manifests BIGINT DEFAULT 0, + total_delete_manifests BIGINT DEFAULT 0, + scanned_data_manifests BIGINT DEFAULT 0, + scanned_delete_manifests BIGINT DEFAULT 0, + skipped_data_manifests BIGINT DEFAULT 0, + skipped_delete_manifests BIGINT DEFAULT 0, + skipped_data_files BIGINT DEFAULT 0, + skipped_delete_files BIGINT DEFAULT 0, + total_planning_duration_ms BIGINT DEFAULT 0, + + -- Equality/positional delete metrics + equality_delete_files BIGINT DEFAULT 0, + positional_delete_files BIGINT DEFAULT 0, + indexed_delete_files BIGINT DEFAULT 0, + total_delete_file_size_bytes BIGINT DEFAULT 0, + + -- Additional metadata (for extensibility) + metadata JSONB DEFAULT '{}'::JSONB, + + PRIMARY KEY (realm_id, report_id) +); + +COMMENT ON TABLE scan_metrics_report IS 'Scan metrics reports as first-class entities'; +COMMENT ON COLUMN scan_metrics_report.report_id IS 'Unique identifier for the report'; +COMMENT ON COLUMN scan_metrics_report.realm_id IS 'Realm ID for multi-tenancy'; +COMMENT ON COLUMN scan_metrics_report.catalog_id IS 'Catalog ID'; +COMMENT ON COLUMN scan_metrics_report.otel_trace_id IS 'OpenTelemetry trace ID from HTTP headers'; +COMMENT ON COLUMN scan_metrics_report.report_trace_id IS 'Trace ID from report metadata'; + +-- Indexes for scan_metrics_report +-- Note: Additional indexes for query patterns (by table, trace_id, principal) can be added +-- when analytics APIs are introduced. Currently only timestamp index is needed for retention cleanup. +CREATE INDEX IF NOT EXISTS idx_scan_report_timestamp + ON scan_metrics_report(realm_id, timestamp_ms DESC); + +-- Junction table for scan metrics report roles +CREATE TABLE IF NOT EXISTS scan_metrics_report_roles ( + realm_id TEXT NOT NULL, + report_id TEXT NOT NULL, + role_name TEXT NOT NULL, + PRIMARY KEY (realm_id, report_id, role_name), + FOREIGN KEY (realm_id, report_id) REFERENCES scan_metrics_report(realm_id, report_id) ON DELETE CASCADE +); + +COMMENT ON TABLE scan_metrics_report_roles IS 'Activated principal roles for scan metrics reports'; + + +-- Commit Metrics Report Entity Table +CREATE TABLE IF NOT EXISTS commit_metrics_report ( + report_id TEXT NOT NULL, + realm_id TEXT NOT NULL, + catalog_id TEXT NOT NULL, + catalog_name TEXT NOT NULL, + namespace TEXT NOT NULL, + table_name TEXT NOT NULL, + + -- Report metadata + timestamp_ms BIGINT NOT NULL, + principal_name TEXT, + request_id TEXT, + + -- Trace correlation + otel_trace_id TEXT, + otel_span_id TEXT, + report_trace_id TEXT, + + -- Commit context + snapshot_id BIGINT NOT NULL, + sequence_number BIGINT, + operation TEXT NOT NULL, + + -- File metrics + added_data_files BIGINT DEFAULT 0, + removed_data_files BIGINT DEFAULT 0, + total_data_files BIGINT DEFAULT 0, + added_delete_files BIGINT DEFAULT 0, + removed_delete_files BIGINT DEFAULT 0, + total_delete_files BIGINT DEFAULT 0, + + -- Equality delete files + added_equality_delete_files BIGINT DEFAULT 0, + removed_equality_delete_files BIGINT DEFAULT 0, + + -- Positional delete files + added_positional_delete_files BIGINT DEFAULT 0, + removed_positional_delete_files BIGINT DEFAULT 0, + + -- Record metrics + added_records BIGINT DEFAULT 0, + removed_records BIGINT DEFAULT 0, + total_records BIGINT DEFAULT 0, + + -- Size metrics + added_file_size_bytes BIGINT DEFAULT 0, + removed_file_size_bytes BIGINT DEFAULT 0, + total_file_size_bytes BIGINT DEFAULT 0, + + -- Duration and attempts + total_duration_ms BIGINT DEFAULT 0, + attempts INTEGER DEFAULT 1, + + -- Additional metadata (for extensibility) + metadata JSONB DEFAULT '{}'::JSONB, + + PRIMARY KEY (realm_id, report_id) +); + +COMMENT ON TABLE commit_metrics_report IS 'Commit metrics reports as first-class entities'; +COMMENT ON COLUMN commit_metrics_report.report_id IS 'Unique identifier for the report'; +COMMENT ON COLUMN commit_metrics_report.realm_id IS 'Realm ID for multi-tenancy'; +COMMENT ON COLUMN commit_metrics_report.operation IS 'Commit operation type: append, overwrite, delete, replace'; +COMMENT ON COLUMN commit_metrics_report.otel_trace_id IS 'OpenTelemetry trace ID from HTTP headers'; + +-- Indexes for commit_metrics_report +-- Note: Additional indexes for query patterns (by table, trace_id, principal, operation, snapshot) +-- can be added when analytics APIs are introduced. Currently only timestamp index is needed for retention cleanup. +CREATE INDEX IF NOT EXISTS idx_commit_report_timestamp + ON commit_metrics_report(realm_id, timestamp_ms DESC); + +-- Junction table for commit metrics report roles +CREATE TABLE IF NOT EXISTS commit_metrics_report_roles ( + realm_id TEXT NOT NULL, + report_id TEXT NOT NULL, + role_name TEXT NOT NULL, + PRIMARY KEY (realm_id, report_id, role_name), + FOREIGN KEY (realm_id, report_id) REFERENCES commit_metrics_report(realm_id, report_id) ON DELETE CASCADE +); Review Comment: do we really need a new table ? @dimas-b thoughts ? ########## persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReport.java: ########## @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.models; + +import jakarta.annotation.Nullable; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.polaris.immutables.PolarisImmutable; +import org.apache.polaris.persistence.relational.jdbc.DatabaseType; +import org.immutables.value.Value; + +/** Model class for commit_metrics_report table - stores commit metrics as first-class entities. */ +@PolarisImmutable +public interface ModelCommitMetricsReport extends Converter<ModelCommitMetricsReport> { + String TABLE_NAME = "COMMIT_METRICS_REPORT"; + + // Column names + String REPORT_ID = "report_id"; + String REALM_ID = "realm_id"; + String CATALOG_ID = "catalog_id"; + String CATALOG_NAME = "catalog_name"; + String NAMESPACE = "namespace"; + String TABLE_NAME_COL = "table_name"; + String TIMESTAMP_MS = "timestamp_ms"; + String PRINCIPAL_NAME = "principal_name"; + String REQUEST_ID = "request_id"; + String OTEL_TRACE_ID = "otel_trace_id"; + String OTEL_SPAN_ID = "otel_span_id"; + String REPORT_TRACE_ID = "report_trace_id"; + String SNAPSHOT_ID = "snapshot_id"; + String SEQUENCE_NUMBER = "sequence_number"; + String OPERATION = "operation"; + String ADDED_DATA_FILES = "added_data_files"; + String REMOVED_DATA_FILES = "removed_data_files"; + String TOTAL_DATA_FILES = "total_data_files"; + String ADDED_DELETE_FILES = "added_delete_files"; + String REMOVED_DELETE_FILES = "removed_delete_files"; + String TOTAL_DELETE_FILES = "total_delete_files"; + String ADDED_EQUALITY_DELETE_FILES = "added_equality_delete_files"; + String REMOVED_EQUALITY_DELETE_FILES = "removed_equality_delete_files"; + String ADDED_POSITIONAL_DELETE_FILES = "added_positional_delete_files"; + String REMOVED_POSITIONAL_DELETE_FILES = "removed_positional_delete_files"; + String ADDED_RECORDS = "added_records"; + String REMOVED_RECORDS = "removed_records"; + String TOTAL_RECORDS = "total_records"; + String ADDED_FILE_SIZE_BYTES = "added_file_size_bytes"; + String REMOVED_FILE_SIZE_BYTES = "removed_file_size_bytes"; + String TOTAL_FILE_SIZE_BYTES = "total_file_size_bytes"; + String TOTAL_DURATION_MS = "total_duration_ms"; + String ATTEMPTS = "attempts"; + String METADATA = "metadata"; + + List<String> ALL_COLUMNS = + List.of( + REPORT_ID, + REALM_ID, + CATALOG_ID, + CATALOG_NAME, + NAMESPACE, + TABLE_NAME_COL, + TIMESTAMP_MS, + PRINCIPAL_NAME, + REQUEST_ID, + OTEL_TRACE_ID, + OTEL_SPAN_ID, + REPORT_TRACE_ID, + SNAPSHOT_ID, + SEQUENCE_NUMBER, + OPERATION, + ADDED_DATA_FILES, + REMOVED_DATA_FILES, + TOTAL_DATA_FILES, + ADDED_DELETE_FILES, + REMOVED_DELETE_FILES, + TOTAL_DELETE_FILES, + ADDED_EQUALITY_DELETE_FILES, + REMOVED_EQUALITY_DELETE_FILES, + ADDED_POSITIONAL_DELETE_FILES, + REMOVED_POSITIONAL_DELETE_FILES, + ADDED_RECORDS, + REMOVED_RECORDS, + TOTAL_RECORDS, + ADDED_FILE_SIZE_BYTES, + REMOVED_FILE_SIZE_BYTES, + TOTAL_FILE_SIZE_BYTES, + TOTAL_DURATION_MS, + ATTEMPTS, + METADATA); + + // Getters + String getReportId(); + + String getRealmId(); + + String getCatalogId(); + + String getCatalogName(); + + String getNamespace(); + + String getTableName(); + + long getTimestampMs(); + + @Nullable + String getPrincipalName(); + + @Nullable + String getRequestId(); + + @Nullable + String getOtelTraceId(); + + @Nullable + String getOtelSpanId(); + + @Nullable + String getReportTraceId(); + + long getSnapshotId(); + + @Nullable + Long getSequenceNumber(); + + String getOperation(); + + long getAddedDataFiles(); + + long getRemovedDataFiles(); + + long getTotalDataFiles(); + + long getAddedDeleteFiles(); + + long getRemovedDeleteFiles(); + + long getTotalDeleteFiles(); + + long getAddedEqualityDeleteFiles(); + + long getRemovedEqualityDeleteFiles(); + + long getAddedPositionalDeleteFiles(); + + long getRemovedPositionalDeleteFiles(); + + long getAddedRecords(); + + long getRemovedRecords(); + + long getTotalRecords(); + + long getAddedFileSizeBytes(); + + long getRemovedFileSizeBytes(); + + long getTotalFileSizeBytes(); + + long getTotalDurationMs(); + + int getAttempts(); + + @Nullable + String getMetadata(); + + /** + * Returns the activated principal roles associated with this report. This is populated from the + * junction table commit_metrics_report_roles. + */ + @Value.Default + default Set<String> getRoles() { + return Set.of(); + } + + @Override + default ModelCommitMetricsReport fromResultSet(ResultSet rs) throws SQLException { + return ImmutableModelCommitMetricsReport.builder() + .reportId(rs.getString(REPORT_ID)) + .realmId(rs.getString(REALM_ID)) + .catalogId(rs.getString(CATALOG_ID)) + .catalogName(rs.getString(CATALOG_NAME)) + .namespace(rs.getString(NAMESPACE)) + .tableName(rs.getString(TABLE_NAME_COL)) + .timestampMs(rs.getLong(TIMESTAMP_MS)) + .principalName(rs.getString(PRINCIPAL_NAME)) + .requestId(rs.getString(REQUEST_ID)) + .otelTraceId(rs.getString(OTEL_TRACE_ID)) + .otelSpanId(rs.getString(OTEL_SPAN_ID)) + .reportTraceId(rs.getString(REPORT_TRACE_ID)) + .snapshotId(rs.getLong(SNAPSHOT_ID)) + .sequenceNumber(rs.getObject(SEQUENCE_NUMBER, Long.class)) Review Comment: why not `rs.getLong()` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
