This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 372876f02a [#10098] feat(optimizer): support MYSQL and PG for metrics
storage (#164) (#10105)
372876f02a is described below
commit 372876f02a5995b9060ed082418c0ff47a442161
Author: FANNG <[email protected]>
AuthorDate: Mon Mar 2 15:29:28 2026 +0900
[#10098] feat(optimizer): support MYSQL and PG for metrics storage (#164)
(#10105)
## What changes are added in this PR?
- Added JDBC metrics storage support for MySQL and PostgreSQL (H2
already
supported).
- Updated metrics schema creation/validation flow in
JdbcMetricsRepository and
GenericJdbcMetricsRepository.
- Added metrics DDL to schema-1.2.0-*.sql and
upgrade-1.1.0-to-1.2.0-*.sql for
H2/MySQL/PostgreSQL.
- Removed obsolete scripts/h2/optimizer-metrics-schema-1.1.0-h2.sql.
- Refactored tests:
- Introduced shared behavior base test.
- Added H2/MySQL/PostgreSQL integration tests under
integration.test.storage. - Kept Docker tag only on MySQL/PG tests. -
Removed redundant prefix-override tests.
## Why are the changes needed?
- Before this PR, JDBC metrics repository was effectively H2-centric.
- Metrics tables were required by runtime validation, but upgrade
scripts did
not consistently guarantee table creation for upgraded deployments.
- Config behavior needed to be explicit (use
gravitino.optimizer.jdbcMetrics.*
directly, no EntityStore fallback).
- Cross-database behavior needed integration coverage.
Fix: #10098
## Does this PR introduce any user-facing change?
- Yes: JDBC metrics storage now supports MySQL and PostgreSQL in
addition to
H2.
- No new public API.
- No new property keys added.
## How was this patch tested?
./
gradlew :maintenance:optimizer:spotlessApply
:maintenance:optimizer:compileTes
tJava
IT_SPARK_HOME=/Users/fanng/deploy/demo/spark-3.5.3-bin-hadoop3
IT_SPARK_ARGS='--conf
spark.jars=/Users/fanng/deploy/demo/jars/iceberg-spark-
runtime-3.5_2.12-1.9.2.jar,/Users/fanng/deploy/demo/jars/iceberg-aws-bundle-
1.9.2.jar --conf
spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionEx
tensions --conf
spark.sql.catalog.rest=org.apache.iceberg.spark.SparkCatalog
--conf spark.sql.catalog.rest.type=rest --conf
spark.sql.catalog.rest.uri=http://127.0.0.1:9001/iceberg/ --conf
spark.sql.catalog.rest.header.X-Iceberg-Access-Delegation=vended-credentials'
IT_TABLE_IDENTIFIER=rest.ab.a1 GRAVITINO_ENV_IT=true ./
gradlew :maintenance:optimizer:test --tests
'org.apache.gravitino.maintenance.optimizer.integration.test.storage.TestH2Gen
ericJdbcMetricsRepositoryIT' -PskipIT=false
---
maintenance/optimizer/build.gradle.kts | 5 +
.../storage/jdbc/GenericJdbcMetricsRepository.java | 12 +-
.../storage/jdbc/JdbcMetricsRepository.java | 10 +-
.../metrics/storage/jdbc/MySQLMetricsDialect.java | 29 ++++
.../storage/jdbc/PostgreSQLMetricsDialect.java | 29 ++++
.../test/storage/JdbcMetricsRepositoryITUtils.java | 89 ++++++++++++
.../TestH2GenericJdbcMetricsRepositoryIT.java | 151 ++++++++++++++++++++
...ySqlGenericJdbcMetricsRepositoryBehaviorIT.java | 61 ++++++++
...stPostgreSqlGenericJdbcMetricsRepositoryIT.java | 61 ++++++++
.../metrics/TestGravitinoMetricsUpdater.java | 2 +-
...a => BaseGenericJdbcMetricsRepositoryTest.java} | 157 ++-------------------
scripts/h2/optimizer-metrics-schema-1.2.0-h2.sql | 41 ------
scripts/h2/schema-1.2.0-h2.sql | 27 ++++
scripts/h2/upgrade-1.1.0-to-1.2.0-h2.sql | 27 ++++
scripts/mysql/schema-1.2.0-mysql.sql | 24 ++++
scripts/mysql/upgrade-1.1.0-to-1.2.0-mysql.sql | 24 ++++
scripts/postgresql/schema-1.2.0-postgresql.sql | 39 +++++
.../upgrade-1.1.0-to-1.2.0-postgresql.sql | 39 +++++
18 files changed, 636 insertions(+), 191 deletions(-)
diff --git a/maintenance/optimizer/build.gradle.kts
b/maintenance/optimizer/build.gradle.kts
index f986287674..d539cc894b 100644
--- a/maintenance/optimizer/build.gradle.kts
+++ b/maintenance/optimizer/build.gradle.kts
@@ -84,6 +84,11 @@ dependencies {
exclude(group = "org.slf4j", module = "slf4j-log4j12")
}
testImplementation(libs.testcontainers)
+ testImplementation(libs.testcontainers.junit.jupiter)
+ testImplementation(libs.testcontainers.mysql)
+ testImplementation(libs.testcontainers.postgresql)
+ testRuntimeOnly(libs.mysql.driver)
+ testRuntimeOnly(libs.postgresql.driver)
testAnnotationProcessor(libs.lombok)
testCompileOnly(libs.lombok)
diff --git
a/maintenance/optimizer/src/main/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/jdbc/GenericJdbcMetricsRepository.java
b/maintenance/optimizer/src/main/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/jdbc/GenericJdbcMetricsRepository.java
index 3e7b818d3e..9be98ef24c 100644
---
a/maintenance/optimizer/src/main/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/jdbc/GenericJdbcMetricsRepository.java
+++
b/maintenance/optimizer/src/main/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/jdbc/GenericJdbcMetricsRepository.java
@@ -44,10 +44,7 @@ public class GenericJdbcMetricsRepository extends
JdbcMetricsRepository {
@Override
public void initialize(Map<String, String> optimizerProperties) {
- Map<String, String> jdbcProperties =
- MapUtils.getPrefixMap(
- optimizerProperties, OptimizerConfig.OPTIMIZER_PREFIX +
JDBC_METRICS_PREFIX);
- Map<String, String> effectiveJdbcProperties = new
HashMap<>(jdbcProperties);
+ Map<String, String> effectiveJdbcProperties =
buildEffectiveJdbcProperties(optimizerProperties);
String jdbcUrl = effectiveJdbcProperties.getOrDefault(JDBC_URL,
DEFAULT_H2_JDBC_URL);
if (StringUtils.isBlank(jdbcUrl)) {
@@ -63,4 +60,11 @@ public class GenericJdbcMetricsRepository extends
JdbcMetricsRepository {
new DataSourceJdbcConnectionProvider(effectiveJdbcProperties);
initializeStorage(connectionProvider);
}
+
+ private Map<String, String> buildEffectiveJdbcProperties(
+ Map<String, String> optimizerProperties) {
+ return new HashMap<>(
+ MapUtils.getPrefixMap(
+ optimizerProperties, OptimizerConfig.OPTIMIZER_PREFIX +
JDBC_METRICS_PREFIX));
+ }
}
diff --git
a/maintenance/optimizer/src/main/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/jdbc/JdbcMetricsRepository.java
b/maintenance/optimizer/src/main/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/jdbc/JdbcMetricsRepository.java
index d16d0b0963..3ff92e246a 100644
---
a/maintenance/optimizer/src/main/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/jdbc/JdbcMetricsRepository.java
+++
b/maintenance/optimizer/src/main/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/jdbc/JdbcMetricsRepository.java
@@ -65,8 +65,10 @@ public abstract class JdbcMetricsRepository implements
MetricsRepository {
private static final int MAX_METRIC_NAME_LENGTH = 1024;
private static final long MAX_REASONABLE_EPOCH_SECONDS = 9_999_999_999L;
private static final String DATABASE_PRODUCT_H2 = "h2";
+ private static final String DATABASE_PRODUCT_MYSQL = "mysql";
+ private static final String DATABASE_PRODUCT_POSTGRESQL = "postgresql";
private static final String H2_SCHEMA_FILE_NAME =
- "optimizer-metrics-schema-" + ConfigConstants.CURRENT_SCRIPT_VERSION +
"-h2.sql";
+ "schema-" + ConfigConstants.CURRENT_SCRIPT_VERSION + "-h2.sql";
private static final String TABLE_METRICS_TABLE = "table_metrics";
private static final String JOB_METRICS_TABLE = "job_metrics";
private static final Set<String> REQUIRED_TABLE_METRICS_COLUMNS =
@@ -186,6 +188,12 @@ public abstract class JdbcMetricsRepository implements
MetricsRepository {
if (databaseProduct.contains(DATABASE_PRODUCT_H2)) {
return new H2MetricsDialect();
}
+ if (databaseProduct.contains(DATABASE_PRODUCT_MYSQL)) {
+ return new MySQLMetricsDialect();
+ }
+ if (databaseProduct.contains(DATABASE_PRODUCT_POSTGRESQL)) {
+ return new PostgreSQLMetricsDialect();
+ }
throw new IllegalArgumentException(
"Unsupported JDBC database product for metrics repository: " +
databaseProduct);
diff --git
a/maintenance/optimizer/src/main/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/jdbc/MySQLMetricsDialect.java
b/maintenance/optimizer/src/main/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/jdbc/MySQLMetricsDialect.java
new file mode 100644
index 0000000000..7bdb502bc3
--- /dev/null
+++
b/maintenance/optimizer/src/main/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/jdbc/MySQLMetricsDialect.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package
org.apache.gravitino.maintenance.optimizer.updater.metrics.storage.jdbc;
+
+/** MySQL dialect marker for JDBC metrics storage. */
+public class MySQLMetricsDialect implements JdbcMetricsDialect {
+
+ @Override
+ public String name() {
+ return "mysql";
+ }
+}
diff --git
a/maintenance/optimizer/src/main/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/jdbc/PostgreSQLMetricsDialect.java
b/maintenance/optimizer/src/main/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/jdbc/PostgreSQLMetricsDialect.java
new file mode 100644
index 0000000000..fda39a4a50
--- /dev/null
+++
b/maintenance/optimizer/src/main/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/jdbc/PostgreSQLMetricsDialect.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package
org.apache.gravitino.maintenance.optimizer.updater.metrics.storage.jdbc;
+
+/** PostgreSQL dialect marker for JDBC metrics storage. */
+public class PostgreSQLMetricsDialect implements JdbcMetricsDialect {
+
+ @Override
+ public String name() {
+ return "postgresql";
+ }
+}
diff --git
a/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/JdbcMetricsRepositoryITUtils.java
b/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/JdbcMetricsRepositoryITUtils.java
new file mode 100644
index 0000000000..801d03e06d
--- /dev/null
+++
b/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/JdbcMetricsRepositoryITUtils.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.maintenance.optimizer.integration.test.storage;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.gravitino.config.ConfigConstants;
+import org.apache.gravitino.maintenance.optimizer.common.conf.OptimizerConfig;
+import
org.apache.gravitino.maintenance.optimizer.updater.metrics.storage.jdbc.GenericJdbcMetricsRepository;
+import org.apache.gravitino.utils.jdbc.JdbcSqlScriptUtils;
+
+final class JdbcMetricsRepositoryITUtils {
+
+ private JdbcMetricsRepositoryITUtils() {}
+
+ static void initializeSchema(
+ String jdbcUrl, String username, String password, String databaseType) {
+ String schemaSql = loadSchemaSql(databaseType);
+ try (Connection connection = DriverManager.getConnection(jdbcUrl,
username, password)) {
+ JdbcSqlScriptUtils.executeSqlScript(connection, schemaSql);
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to initialize " + databaseType + "
schema for metrics", e);
+ }
+ }
+
+ static String loadSchemaSql(String databaseType) {
+ String gravitinoHome = System.getenv("GRAVITINO_HOME");
+ if (StringUtils.isBlank(gravitinoHome)) {
+ throw new IllegalStateException("GRAVITINO_HOME environment variable
must be set for ITs");
+ }
+ Path root = Path.of(gravitinoHome);
+ Path scriptPath =
+ root.resolve(
+ Path.of(
+ "scripts",
+ databaseType,
+ "schema-" + ConfigConstants.CURRENT_SCRIPT_VERSION + "-" +
databaseType + ".sql"));
+ try {
+ return Files.readString(scriptPath, StandardCharsets.UTF_8);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to load schema script: " +
scriptPath, e);
+ }
+ }
+
+ static Map<String, String> createJdbcMetricsConfigs(
+ String jdbcUrl, String username, String password, String
driverClassName) {
+ return Map.of(
+ OptimizerConfig.OPTIMIZER_PREFIX
+ + GenericJdbcMetricsRepository.JDBC_METRICS_PREFIX
+ + GenericJdbcMetricsRepository.JDBC_URL,
+ jdbcUrl,
+ OptimizerConfig.OPTIMIZER_PREFIX
+ + GenericJdbcMetricsRepository.JDBC_METRICS_PREFIX
+ + GenericJdbcMetricsRepository.JDBC_USER,
+ username,
+ OptimizerConfig.OPTIMIZER_PREFIX
+ + GenericJdbcMetricsRepository.JDBC_METRICS_PREFIX
+ + GenericJdbcMetricsRepository.JDBC_PASSWORD,
+ password,
+ OptimizerConfig.OPTIMIZER_PREFIX
+ + GenericJdbcMetricsRepository.JDBC_METRICS_PREFIX
+ + GenericJdbcMetricsRepository.JDBC_DRIVER,
+ driverClassName);
+ }
+}
diff --git
a/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestH2GenericJdbcMetricsRepositoryIT.java
b/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestH2GenericJdbcMetricsRepositoryIT.java
new file mode 100644
index 0000000000..1eabae2134
--- /dev/null
+++
b/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestH2GenericJdbcMetricsRepositoryIT.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.maintenance.optimizer.integration.test.storage;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import org.apache.gravitino.NameIdentifier;
+import org.apache.gravitino.maintenance.optimizer.common.conf.OptimizerConfig;
+import
org.apache.gravitino.maintenance.optimizer.updater.metrics.storage.BaseGenericJdbcMetricsRepositoryTest;
+import
org.apache.gravitino.maintenance.optimizer.updater.metrics.storage.MetricRecord;
+import
org.apache.gravitino.maintenance.optimizer.updater.metrics.storage.MetricRecordImpl;
+import
org.apache.gravitino.maintenance.optimizer.updater.metrics.storage.TableMetricWriteRequest;
+import
org.apache.gravitino.maintenance.optimizer.updater.metrics.storage.jdbc.GenericJdbcMetricsRepository;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+class TestH2GenericJdbcMetricsRepositoryIT extends
BaseGenericJdbcMetricsRepositoryTest {
+ private static final String H2_USER = "sa";
+ private static final String H2_PASSWORD = "";
+ private static final String H2_DRIVER = "org.h2.Driver";
+
+ private String jdbcUrl;
+
+ @BeforeAll
+ void setUp() throws IOException {
+ Path testDir = Files.createTempDirectory("optimizer-h2-metrics");
+ jdbcUrl = "jdbc:h2:file:" + testDir.resolve("metrics.db") + ";MODE=MYSQL";
+ initializeSchema(jdbcUrl);
+ storage = new GenericJdbcMetricsRepository();
+ storage.initialize(createJdbcConfigs(jdbcUrl));
+ storage.cleanupTableMetricsBefore(MAX_REASONABLE_EPOCH_SECONDS);
+ storage.cleanupJobMetricsBefore(MAX_REASONABLE_EPOCH_SECONDS);
+ }
+
+ @AfterAll
+ void tearDown() {
+ storage.cleanupTableMetricsBefore(MAX_REASONABLE_EPOCH_SECONDS);
+ storage.cleanupJobMetricsBefore(MAX_REASONABLE_EPOCH_SECONDS);
+ storage.close();
+ }
+
+ @Test
+ void testInitializeTwiceFails() {
+ GenericJdbcMetricsRepository repository = new
GenericJdbcMetricsRepository();
+ String initTwiceJdbcUrl = withSuffix(jdbcUrl, "_init_twice");
+ initializeSchema(initTwiceJdbcUrl);
+ repository.initialize(createJdbcConfigs(initTwiceJdbcUrl));
+ try {
+ IllegalStateException e =
+ Assertions.assertThrows(
+ IllegalStateException.class,
+ () ->
repository.initialize(createJdbcConfigs(initTwiceJdbcUrl)));
+ Assertions.assertTrue(e.getMessage().contains("already been
initialized"));
+ } finally {
+ cleanupAndClose(repository);
+ }
+ }
+
+ @Test
+ void testInitializeWithoutSchemaAutoCreatesForH2() {
+ String autoCreateSchemaJdbcUrl = withSuffix(jdbcUrl,
"_auto_create_schema");
+ GenericJdbcMetricsRepository repository =
createInitializedRepository(autoCreateSchemaJdbcUrl);
+ try {
+ NameIdentifier tableId = NameIdentifier.of("catalog", "db",
"auto_create_table");
+ long now = currentEpochSeconds();
+ repository.storeTableMetrics(
+ List.of(
+ new TableMetricWriteRequest(
+ tableId, "row_count", Optional.empty(), new
MetricRecordImpl(now, "12"))));
+
+ Map<String, List<MetricRecord>> metrics =
+ repository.getTableMetrics(tableId, now - 1, now + 1);
+ Assertions.assertTrue(metrics.containsKey("row_count"));
+ Assertions.assertEquals(List.of("12"),
getMetricValues(metrics.get("row_count")));
+ } finally {
+ cleanupAndClose(repository);
+ }
+ }
+
+ private Map<String, String> createJdbcConfigs(String jdbcUrl) {
+ return Map.of(
+ OptimizerConfig.OPTIMIZER_PREFIX
+ + GenericJdbcMetricsRepository.JDBC_METRICS_PREFIX
+ + GenericJdbcMetricsRepository.JDBC_URL,
+ jdbcUrl,
+ OptimizerConfig.OPTIMIZER_PREFIX
+ + GenericJdbcMetricsRepository.JDBC_METRICS_PREFIX
+ + GenericJdbcMetricsRepository.JDBC_USER,
+ H2_USER,
+ OptimizerConfig.OPTIMIZER_PREFIX
+ + GenericJdbcMetricsRepository.JDBC_METRICS_PREFIX
+ + GenericJdbcMetricsRepository.JDBC_PASSWORD,
+ H2_PASSWORD,
+ OptimizerConfig.OPTIMIZER_PREFIX
+ + GenericJdbcMetricsRepository.JDBC_METRICS_PREFIX
+ + GenericJdbcMetricsRepository.JDBC_DRIVER,
+ H2_DRIVER);
+ }
+
+ private void initializeSchema(String jdbcUrl) {
+ JdbcMetricsRepositoryITUtils.initializeSchema(jdbcUrl, H2_USER,
H2_PASSWORD, "h2");
+ }
+
+ private String withSuffix(String baseJdbcUrl, String suffix) {
+ int optionsSeparator = baseJdbcUrl.indexOf(';');
+ if (optionsSeparator < 0) {
+ return baseJdbcUrl + suffix;
+ }
+
+ return baseJdbcUrl.substring(0, optionsSeparator)
+ + suffix
+ + baseJdbcUrl.substring(optionsSeparator);
+ }
+
+ private GenericJdbcMetricsRepository createInitializedRepository(String
jdbcUrl) {
+ GenericJdbcMetricsRepository repository = new
GenericJdbcMetricsRepository();
+ repository.initialize(createJdbcConfigs(jdbcUrl));
+ return repository;
+ }
+
+ private void cleanupAndClose(GenericJdbcMetricsRepository repository) {
+ repository.cleanupTableMetricsBefore(MAX_REASONABLE_EPOCH_SECONDS);
+ repository.cleanupJobMetricsBefore(MAX_REASONABLE_EPOCH_SECONDS);
+ repository.close();
+ }
+}
diff --git
a/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestMySqlGenericJdbcMetricsRepositoryBehaviorIT.java
b/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestMySqlGenericJdbcMetricsRepositoryBehaviorIT.java
new file mode 100644
index 0000000000..da64f064c3
--- /dev/null
+++
b/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestMySqlGenericJdbcMetricsRepositoryBehaviorIT.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.maintenance.optimizer.integration.test.storage;
+
+import
org.apache.gravitino.maintenance.optimizer.updater.metrics.storage.BaseGenericJdbcMetricsRepositoryTest;
+import
org.apache.gravitino.maintenance.optimizer.updater.metrics.storage.jdbc.GenericJdbcMetricsRepository;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.TestInstance;
+import org.testcontainers.containers.MySQLContainer;
+
+@Tag("gravitino-docker-test")
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+class TestMySqlGenericJdbcMetricsRepositoryBehaviorIT extends
BaseGenericJdbcMetricsRepositoryTest {
+
+ private MySQLContainer<?> mysql;
+
+ @BeforeAll
+ void setUp() {
+ mysql = new MySQLContainer<>("mysql:8.0.33");
+ mysql.start();
+ JdbcMetricsRepositoryITUtils.initializeSchema(
+ mysql.getJdbcUrl(), mysql.getUsername(), mysql.getPassword(), "mysql");
+
+ storage = new GenericJdbcMetricsRepository();
+ storage.initialize(
+ JdbcMetricsRepositoryITUtils.createJdbcMetricsConfigs(
+ mysql.getJdbcUrl(),
+ mysql.getUsername(),
+ mysql.getPassword(),
+ mysql.getDriverClassName()));
+ }
+
+ @AfterAll
+ void tearDown() {
+ storage.cleanupTableMetricsBefore(MAX_REASONABLE_EPOCH_SECONDS);
+ storage.cleanupJobMetricsBefore(MAX_REASONABLE_EPOCH_SECONDS);
+ storage.close();
+ if (mysql != null) {
+ mysql.stop();
+ }
+ }
+}
diff --git
a/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestPostgreSqlGenericJdbcMetricsRepositoryIT.java
b/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestPostgreSqlGenericJdbcMetricsRepositoryIT.java
new file mode 100644
index 0000000000..8f29479be0
--- /dev/null
+++
b/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestPostgreSqlGenericJdbcMetricsRepositoryIT.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.maintenance.optimizer.integration.test.storage;
+
+import
org.apache.gravitino.maintenance.optimizer.updater.metrics.storage.BaseGenericJdbcMetricsRepositoryTest;
+import
org.apache.gravitino.maintenance.optimizer.updater.metrics.storage.jdbc.GenericJdbcMetricsRepository;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.TestInstance;
+import org.testcontainers.containers.PostgreSQLContainer;
+
+@Tag("gravitino-docker-test")
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+class TestPostgreSqlGenericJdbcMetricsRepositoryIT extends
BaseGenericJdbcMetricsRepositoryTest {
+
+ private PostgreSQLContainer<?> postgres;
+
+ @BeforeAll
+ void setUp() {
+ postgres = new PostgreSQLContainer<>("postgres:13");
+ postgres.start();
+ JdbcMetricsRepositoryITUtils.initializeSchema(
+ postgres.getJdbcUrl(), postgres.getUsername(), postgres.getPassword(),
"postgresql");
+
+ storage = new GenericJdbcMetricsRepository();
+ storage.initialize(
+ JdbcMetricsRepositoryITUtils.createJdbcMetricsConfigs(
+ postgres.getJdbcUrl(),
+ postgres.getUsername(),
+ postgres.getPassword(),
+ postgres.getDriverClassName()));
+ }
+
+ @AfterAll
+ void tearDown() {
+ storage.cleanupTableMetricsBefore(MAX_REASONABLE_EPOCH_SECONDS);
+ storage.cleanupJobMetricsBefore(MAX_REASONABLE_EPOCH_SECONDS);
+ storage.close();
+ if (postgres != null) {
+ postgres.stop();
+ }
+ }
+}
diff --git
a/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/TestGravitinoMetricsUpdater.java
b/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/TestGravitinoMetricsUpdater.java
index 57411eedf8..6fd3ff10ab 100644
---
a/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/TestGravitinoMetricsUpdater.java
+++
b/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/TestGravitinoMetricsUpdater.java
@@ -162,7 +162,7 @@ class TestGravitinoMetricsUpdater {
@Test
void testInitializeWithJdbcConfigStillUsesGenericJdbcRepository() throws
Exception {
GravitinoMetricsUpdater updater = new GravitinoMetricsUpdater();
- String jdbcUrl =
"jdbc:h2:mem:test_metrics_updater_repo_type;DB_CLOSE_DELAY=-1";
+ String jdbcUrl =
"jdbc:h2:mem:test_metrics_updater_repo_type;DB_CLOSE_DELAY=-1;MODE=MYSQL";
OptimizerConfig config =
new OptimizerConfig(
Map.of(
diff --git
a/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/TestGenericJdbcMetricsRepository.java
b/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/BaseGenericJdbcMetricsRepositoryTest.java
similarity index 70%
rename from
maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/TestGenericJdbcMetricsRepository.java
rename to
maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/BaseGenericJdbcMetricsRepositoryTest.java
index 6c03b4f41c..b1f5f33970 100644
---
a/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/TestGenericJdbcMetricsRepository.java
+++
b/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/BaseGenericJdbcMetricsRepositoryTest.java
@@ -19,55 +19,25 @@
package org.apache.gravitino.maintenance.optimizer.updater.metrics.storage;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
import java.time.Instant;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import org.apache.commons.lang3.StringUtils;
import org.apache.gravitino.NameIdentifier;
-import org.apache.gravitino.config.ConfigConstants;
-import org.apache.gravitino.maintenance.optimizer.common.conf.OptimizerConfig;
import
org.apache.gravitino.maintenance.optimizer.updater.metrics.storage.jdbc.GenericJdbcMetricsRepository;
-import org.apache.gravitino.utils.jdbc.JdbcSqlScriptUtils;
-import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestInstance;
-
-@TestInstance(TestInstance.Lifecycle.PER_CLASS)
-class TestGenericJdbcMetricsRepository {
- private static final long MAX_REASONABLE_EPOCH_SECONDS = 9_999_999_999L;
- private GenericJdbcMetricsRepository storage;
- private String jdbcUrl;
-
- @BeforeAll
- void setUp() throws IOException {
- cleanupLegacyDataFiles();
- Path testDir = Files.createTempDirectory("optimizer-h2-metrics");
- jdbcUrl = "jdbc:h2:file:" + testDir.resolve("metrics.db");
- initializeSchema(jdbcUrl);
- storage = new GenericJdbcMetricsRepository();
- storage.initialize(createJdbcConfigs(jdbcUrl));
- storage.cleanupTableMetricsBefore(MAX_REASONABLE_EPOCH_SECONDS);
- storage.cleanupJobMetricsBefore(MAX_REASONABLE_EPOCH_SECONDS);
- }
- @AfterAll
- void tearDown() {
- storage.cleanupTableMetricsBefore(MAX_REASONABLE_EPOCH_SECONDS);
- storage.cleanupJobMetricsBefore(MAX_REASONABLE_EPOCH_SECONDS);
- storage.close();
- }
+/**
+ * Shared behavior tests for {@link GenericJdbcMetricsRepository}.
+ *
+ * <p>Environment setup (H2/MySQL/PostgreSQL) is provided by subclasses.
+ */
+public abstract class BaseGenericJdbcMetricsRepositoryTest {
+
+ protected static final long MAX_REASONABLE_EPOCH_SECONDS = 9_999_999_999L;
+ protected GenericJdbcMetricsRepository storage;
@Test
void testStoreAndRetrieveTableMetricsWithNullPartition() {
@@ -100,12 +70,9 @@ class TestGenericJdbcMetricsRepository {
storage.getTableMetrics(nameIdentifier, 0, Long.MAX_VALUE);
Assertions.assertEquals(2, metrics.size());
-
Assertions.assertTrue(metrics.containsKey("metric1"));
Assertions.assertEquals(Arrays.asList("value1"),
getMetricValues(metrics.get("metric1")));
-
Assertions.assertTrue(metrics.containsKey("metric2"));
- Assertions.assertEquals(2, getMetricValues(metrics.get("metric2")).size());
Assertions.assertEquals(
Arrays.asList("value1", "value2"),
getMetricValues(metrics.get("metric2")));
}
@@ -128,7 +95,6 @@ class TestGenericJdbcMetricsRepository {
Map<String, List<MetricRecord>> metrics =
storage.getPartitionMetrics(nameIdentifier, partition1, 0,
Long.MAX_VALUE);
-
Assertions.assertEquals(1, metrics.size());
Assertions.assertTrue(metrics.containsKey("metric"));
Assertions.assertEquals(Arrays.asList("value1"),
getMetricValues(metrics.get("metric")));
@@ -199,19 +165,6 @@ class TestGenericJdbcMetricsRepository {
Assertions.assertEquals(List.of("v1"),
getMetricValues(jobMetrics.get("job_metric")));
}
- @Test
- void testInitializeTwiceFails() {
- GenericJdbcMetricsRepository repository = new
GenericJdbcMetricsRepository();
- initializeSchema(jdbcUrl + "_init_twice");
- repository.initialize(createJdbcConfigs(jdbcUrl + "_init_twice"));
-
- IllegalStateException e =
- Assertions.assertThrows(
- IllegalStateException.class,
- () -> repository.initialize(createJdbcConfigs(jdbcUrl +
"_init_twice")));
- Assertions.assertTrue(e.getMessage().contains("already been initialized"));
- }
-
@Test
void testInvalidTimeWindowFailsFast() {
NameIdentifier id = NameIdentifier.of("catalog", "db", "table");
@@ -351,32 +304,11 @@ class TestGenericJdbcMetricsRepository {
Assertions.assertEquals(longValue,
jobMetrics.get("metric_long").get(0).getValue());
}
- @Test
- void testInitializeWithoutSchemaAutoCreatesForH2() {
- GenericJdbcMetricsRepository repository = new
GenericJdbcMetricsRepository();
- String autoCreateSchemaJdbcUrl = jdbcUrl + "_auto_create_schema";
- repository.initialize(createJdbcConfigs(autoCreateSchemaJdbcUrl));
-
- NameIdentifier tableId = NameIdentifier.of("catalog", "db",
"auto_create_table");
- long now = currentEpochSeconds();
- repository.storeTableMetrics(
- List.of(
- new TableMetricWriteRequest(
- tableId, "row_count", Optional.empty(), new
MetricRecordImpl(now, "12"))));
-
- Map<String, List<MetricRecord>> metrics =
repository.getTableMetrics(tableId, now - 1, now + 1);
- Assertions.assertTrue(metrics.containsKey("row_count"));
- Assertions.assertEquals(List.of("12"),
getMetricValues(metrics.get("row_count")));
- repository.cleanupTableMetricsBefore(MAX_REASONABLE_EPOCH_SECONDS);
- repository.cleanupJobMetricsBefore(MAX_REASONABLE_EPOCH_SECONDS);
- repository.close();
- }
-
- private List<String> getMetricValues(List<MetricRecord> metrics) {
+ protected List<String> getMetricValues(List<MetricRecord> metrics) {
return metrics.stream().map(MetricRecord::getValue).toList();
}
- private void storeTableMetric(
+ protected void storeTableMetric(
NameIdentifier nameIdentifier,
String metricName,
Optional<String> partition,
@@ -385,75 +317,12 @@ class TestGenericJdbcMetricsRepository {
List.of(new TableMetricWriteRequest(nameIdentifier, metricName,
partition, metric)));
}
- private void storeJobMetric(
+ protected void storeJobMetric(
NameIdentifier nameIdentifier, String metricName, MetricRecord metric) {
storage.storeJobMetrics(List.of(new JobMetricWriteRequest(nameIdentifier,
metricName, metric)));
}
- private long currentEpochSeconds() {
+ protected long currentEpochSeconds() {
return Instant.now().getEpochSecond();
}
-
- private void cleanupLegacyDataFiles() throws IOException {
- deleteIfExists("data/metrics.db.mv.db");
- deleteIfExists("data/metrics.db.trace.db");
- deleteIfExists("maintenance/optimizer/data/metrics.db.mv.db");
- deleteIfExists("maintenance/optimizer/data/metrics.db.trace.db");
- deleteIfExists("maintenance/optimizer/metrics_db.mv.db");
- deleteIfExists("maintenance/optimizer/metrics_db.trace.db");
- deleteIfExists("metrics_db.mv.db");
- deleteIfExists("metrics_db.trace.db");
- }
-
- private void deleteIfExists(String filePath) throws IOException {
- Files.deleteIfExists(Path.of(filePath));
- }
-
- private Map<String, String> createJdbcConfigs(String jdbcUrl) {
- return Map.of(
- OptimizerConfig.OPTIMIZER_PREFIX
- + GenericJdbcMetricsRepository.JDBC_METRICS_PREFIX
- + GenericJdbcMetricsRepository.JDBC_URL,
- jdbcUrl,
- OptimizerConfig.OPTIMIZER_PREFIX
- + GenericJdbcMetricsRepository.JDBC_METRICS_PREFIX
- + GenericJdbcMetricsRepository.JDBC_USER,
- "sa",
- OptimizerConfig.OPTIMIZER_PREFIX
- + GenericJdbcMetricsRepository.JDBC_METRICS_PREFIX
- + GenericJdbcMetricsRepository.JDBC_PASSWORD,
- "",
- OptimizerConfig.OPTIMIZER_PREFIX
- + GenericJdbcMetricsRepository.JDBC_METRICS_PREFIX
- + GenericJdbcMetricsRepository.JDBC_DRIVER,
- "org.h2.Driver");
- }
-
- private void initializeSchema(String jdbcUrl) {
- String schemaSql = loadSchemaSql();
- try (Connection conn = DriverManager.getConnection(jdbcUrl, "sa", "")) {
- JdbcSqlScriptUtils.executeSqlScript(conn, schemaSql);
- } catch (SQLException e) {
- throw new RuntimeException("Failed to initialize test schema for
metrics", e);
- }
- }
-
- private String loadSchemaSql() {
- String gravitinoHome = System.getenv("GRAVITINO_HOME");
- if (StringUtils.isBlank(gravitinoHome)) {
- throw new RuntimeException("GRAVITINO_HOME environment variable is not
set");
- }
-
- Path scriptPath =
- Paths.get(
- gravitinoHome,
- "scripts",
- "h2",
- "optimizer-metrics-schema-" +
ConfigConstants.CURRENT_SCRIPT_VERSION + "-h2.sql");
- try {
- return Files.readString(scriptPath, StandardCharsets.UTF_8);
- } catch (IOException e) {
- throw new RuntimeException("Failed to load schema script: " +
scriptPath, e);
- }
- }
}
diff --git a/scripts/h2/optimizer-metrics-schema-1.2.0-h2.sql
b/scripts/h2/optimizer-metrics-schema-1.2.0-h2.sql
deleted file mode 100644
index 1c81c024ec..0000000000
--- a/scripts/h2/optimizer-metrics-schema-1.2.0-h2.sql
+++ /dev/null
@@ -1,41 +0,0 @@
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,
--- software distributed under the License is distributed on an
--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- KIND, either express or implied. See the License for the
--- specific language governing permissions and limitations
--- under the License.
-
--- Optimizer metrics schema for H2
-CREATE TABLE IF NOT EXISTS table_metrics (
- id BIGINT AUTO_INCREMENT PRIMARY KEY,
- table_identifier VARCHAR(1024) NOT NULL,
- metric_name VARCHAR(1024) NOT NULL,
- table_partition VARCHAR(1024),
- metric_ts BIGINT NOT NULL,
- metric_value VARCHAR(1024) NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS job_metrics (
- id BIGINT AUTO_INCREMENT PRIMARY KEY,
- job_identifier VARCHAR(1024) NOT NULL,
- metric_name VARCHAR(1024) NOT NULL,
- metric_ts BIGINT NOT NULL,
- metric_value VARCHAR(1024) NOT NULL
-);
-
-CREATE INDEX IF NOT EXISTS idx_table_metrics_metric_ts ON
table_metrics(metric_ts);
-CREATE INDEX IF NOT EXISTS idx_job_metrics_metric_ts ON job_metrics(metric_ts);
-CREATE INDEX IF NOT EXISTS idx_table_metrics_composite
- ON table_metrics(table_identifier, table_partition, metric_ts);
-CREATE INDEX IF NOT EXISTS idx_job_metrics_identifier_metric_ts
- ON job_metrics(job_identifier, metric_ts);
diff --git a/scripts/h2/schema-1.2.0-h2.sql b/scripts/h2/schema-1.2.0-h2.sql
index 9bf2ea2892..1a39474c31 100644
--- a/scripts/h2/schema-1.2.0-h2.sql
+++ b/scripts/h2/schema-1.2.0-h2.sql
@@ -510,3 +510,30 @@ CREATE TABLE IF NOT EXISTS partition_statistic_meta (
);
CREATE INDEX IF NOT EXISTS idx_table_partition ON
partition_statistic_meta(table_id, partition_name);
+
+-- Optimizer metrics schema
+CREATE TABLE IF NOT EXISTS `table_metrics` (
+ `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment
id',
+ `table_identifier` VARCHAR(1024) NOT NULL COMMENT 'normalized table
identifier',
+ `metric_name` VARCHAR(1024) NOT NULL COMMENT 'metric name',
+ `table_partition` VARCHAR(1024) DEFAULT NULL COMMENT 'normalized partition
identifier',
+ `metric_ts` BIGINT(20) NOT NULL COMMENT 'metric timestamp in epoch
seconds',
+ `metric_value` VARCHAR(1024) NOT NULL COMMENT 'metric value payload',
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB COMMENT='optimizer table metrics';
+
+CREATE TABLE IF NOT EXISTS `job_metrics` (
+ `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment
id',
+ `job_identifier` VARCHAR(1024) NOT NULL COMMENT 'normalized job
identifier',
+ `metric_name` VARCHAR(1024) NOT NULL COMMENT 'metric name',
+ `metric_ts` BIGINT(20) NOT NULL COMMENT 'metric timestamp in epoch
seconds',
+ `metric_value` VARCHAR(1024) NOT NULL COMMENT 'metric value payload',
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB COMMENT='optimizer job metrics';
+
+CREATE INDEX IF NOT EXISTS `idx_table_metrics_metric_ts` ON
`table_metrics`(`metric_ts`);
+CREATE INDEX IF NOT EXISTS `idx_job_metrics_metric_ts` ON
`job_metrics`(`metric_ts`);
+CREATE INDEX IF NOT EXISTS `idx_table_metrics_composite`
+ ON `table_metrics`(`table_identifier`, `table_partition`, `metric_ts`);
+CREATE INDEX IF NOT EXISTS `idx_job_metrics_identifier_metric_ts`
+ ON `job_metrics`(`job_identifier`, `metric_ts`);
diff --git a/scripts/h2/upgrade-1.1.0-to-1.2.0-h2.sql
b/scripts/h2/upgrade-1.1.0-to-1.2.0-h2.sql
index 65b05958fe..63eb8cec17 100644
--- a/scripts/h2/upgrade-1.1.0-to-1.2.0-h2.sql
+++ b/scripts/h2/upgrade-1.1.0-to-1.2.0-h2.sql
@@ -81,3 +81,30 @@ CREATE TABLE IF NOT EXISTS partition_statistic_meta (
);
CREATE INDEX IF NOT EXISTS idx_table_partition ON
partition_statistic_meta(table_id, partition_name);
+
+-- Add optimizer metrics storage tables
+CREATE TABLE IF NOT EXISTS `table_metrics` (
+ `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment
id',
+ `table_identifier` VARCHAR(1024) NOT NULL COMMENT 'normalized table
identifier',
+ `metric_name` VARCHAR(1024) NOT NULL COMMENT 'metric name',
+ `table_partition` VARCHAR(1024) DEFAULT NULL COMMENT 'normalized partition
identifier',
+ `metric_ts` BIGINT(20) NOT NULL COMMENT 'metric timestamp in epoch
seconds',
+ `metric_value` VARCHAR(1024) NOT NULL COMMENT 'metric value payload',
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB COMMENT='optimizer table metrics';
+
+CREATE TABLE IF NOT EXISTS `job_metrics` (
+ `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment
id',
+ `job_identifier` VARCHAR(1024) NOT NULL COMMENT 'normalized job
identifier',
+ `metric_name` VARCHAR(1024) NOT NULL COMMENT 'metric name',
+ `metric_ts` BIGINT(20) NOT NULL COMMENT 'metric timestamp in epoch
seconds',
+ `metric_value` VARCHAR(1024) NOT NULL COMMENT 'metric value payload',
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB COMMENT='optimizer job metrics';
+
+CREATE INDEX IF NOT EXISTS `idx_table_metrics_metric_ts` ON
`table_metrics`(`metric_ts`);
+CREATE INDEX IF NOT EXISTS `idx_job_metrics_metric_ts` ON
`job_metrics`(`metric_ts`);
+CREATE INDEX IF NOT EXISTS `idx_table_metrics_composite`
+ ON `table_metrics`(`table_identifier`, `table_partition`, `metric_ts`);
+CREATE INDEX IF NOT EXISTS `idx_job_metrics_identifier_metric_ts`
+ ON `job_metrics`(`job_identifier`, `metric_ts`);
diff --git a/scripts/mysql/schema-1.2.0-mysql.sql
b/scripts/mysql/schema-1.2.0-mysql.sql
index e0fa45cd39..12eac057c4 100644
--- a/scripts/mysql/schema-1.2.0-mysql.sql
+++ b/scripts/mysql/schema-1.2.0-mysql.sql
@@ -500,3 +500,27 @@ CREATE TABLE IF NOT EXISTS `partition_statistic_meta` (
PRIMARY KEY (`table_id`, `partition_name`(255), `statistic_name`),
KEY `idx_table_partition` (`table_id`, `partition_name`(255))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'partition
statistics metadata';
+
+-- Optimizer metrics schema
+CREATE TABLE IF NOT EXISTS `table_metrics` (
+ `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment
id',
+ `table_identifier` VARCHAR(1024) NOT NULL COMMENT 'normalized table
identifier',
+ `metric_name` VARCHAR(1024) NOT NULL COMMENT 'metric name',
+ `table_partition` VARCHAR(1024) DEFAULT NULL COMMENT 'normalized partition
identifier',
+ `metric_ts` BIGINT(20) NOT NULL COMMENT 'metric timestamp in epoch
seconds',
+ `metric_value` VARCHAR(1024) NOT NULL COMMENT 'metric value payload',
+ PRIMARY KEY (`id`),
+ KEY `idx_table_metrics_metric_ts` (`metric_ts`),
+ KEY `idx_table_metrics_composite` (`table_identifier`(255),
`table_partition`(255), `metric_ts`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'optimizer
table metrics';
+
+CREATE TABLE IF NOT EXISTS `job_metrics` (
+ `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment
id',
+ `job_identifier` VARCHAR(1024) NOT NULL COMMENT 'normalized job
identifier',
+ `metric_name` VARCHAR(1024) NOT NULL COMMENT 'metric name',
+ `metric_ts` BIGINT(20) NOT NULL COMMENT 'metric timestamp in epoch
seconds',
+ `metric_value` VARCHAR(1024) NOT NULL COMMENT 'metric value payload',
+ PRIMARY KEY (`id`),
+ KEY `idx_job_metrics_metric_ts` (`metric_ts`),
+ KEY `idx_job_metrics_identifier_metric_ts` (`job_identifier`(255),
`metric_ts`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'optimizer
job metrics';
diff --git a/scripts/mysql/upgrade-1.1.0-to-1.2.0-mysql.sql
b/scripts/mysql/upgrade-1.1.0-to-1.2.0-mysql.sql
index 0de595267f..716302e48c 100644
--- a/scripts/mysql/upgrade-1.1.0-to-1.2.0-mysql.sql
+++ b/scripts/mysql/upgrade-1.1.0-to-1.2.0-mysql.sql
@@ -80,3 +80,27 @@ CREATE TABLE IF NOT EXISTS `partition_statistic_meta` (
PRIMARY KEY (`table_id`, `partition_name`(255), `statistic_name`),
KEY `idx_table_partition` (`table_id`, `partition_name`(255))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'partition
statistics metadata';
+
+-- Add optimizer metrics storage tables
+CREATE TABLE IF NOT EXISTS `table_metrics` (
+ `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment
id',
+ `table_identifier` VARCHAR(1024) NOT NULL COMMENT 'normalized table
identifier',
+ `metric_name` VARCHAR(1024) NOT NULL COMMENT 'metric name',
+ `table_partition` VARCHAR(1024) DEFAULT NULL COMMENT 'normalized partition
identifier',
+ `metric_ts` BIGINT(20) NOT NULL COMMENT 'metric timestamp in epoch
seconds',
+ `metric_value` VARCHAR(1024) NOT NULL COMMENT 'metric value payload',
+ PRIMARY KEY (`id`),
+ KEY `idx_table_metrics_metric_ts` (`metric_ts`),
+ KEY `idx_table_metrics_composite` (`table_identifier`(255),
`table_partition`(255), `metric_ts`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'optimizer
table metrics';
+
+CREATE TABLE IF NOT EXISTS `job_metrics` (
+ `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment
id',
+ `job_identifier` VARCHAR(1024) NOT NULL COMMENT 'normalized job
identifier',
+ `metric_name` VARCHAR(1024) NOT NULL COMMENT 'metric name',
+ `metric_ts` BIGINT(20) NOT NULL COMMENT 'metric timestamp in epoch
seconds',
+ `metric_value` VARCHAR(1024) NOT NULL COMMENT 'metric value payload',
+ PRIMARY KEY (`id`),
+ KEY `idx_job_metrics_metric_ts` (`metric_ts`),
+ KEY `idx_job_metrics_identifier_metric_ts` (`job_identifier`(255),
`metric_ts`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'optimizer
job metrics';
diff --git a/scripts/postgresql/schema-1.2.0-postgresql.sql
b/scripts/postgresql/schema-1.2.0-postgresql.sql
index 6c2c189504..359aa7ef85 100644
--- a/scripts/postgresql/schema-1.2.0-postgresql.sql
+++ b/scripts/postgresql/schema-1.2.0-postgresql.sql
@@ -884,3 +884,42 @@ COMMENT ON COLUMN partition_statistic_meta.statistic_value
IS 'statistic value a
COMMENT ON COLUMN partition_statistic_meta.audit_info IS 'audit information as
JSON';
COMMENT ON COLUMN partition_statistic_meta.created_at IS 'creation timestamp
in milliseconds';
COMMENT ON COLUMN partition_statistic_meta.updated_at IS 'last update
timestamp in milliseconds';
+
+-- Optimizer metrics schema
+CREATE TABLE IF NOT EXISTS table_metrics (
+ id BIGSERIAL PRIMARY KEY,
+ table_identifier VARCHAR(1024) NOT NULL,
+ metric_name VARCHAR(1024) NOT NULL,
+ table_partition VARCHAR(1024),
+ metric_ts BIGINT NOT NULL,
+ metric_value VARCHAR(1024) NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS job_metrics (
+ id BIGSERIAL PRIMARY KEY,
+ job_identifier VARCHAR(1024) NOT NULL,
+ metric_name VARCHAR(1024) NOT NULL,
+ metric_ts BIGINT NOT NULL,
+ metric_value VARCHAR(1024) NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_table_metrics_metric_ts ON
table_metrics(metric_ts);
+CREATE INDEX IF NOT EXISTS idx_job_metrics_metric_ts ON job_metrics(metric_ts);
+CREATE INDEX IF NOT EXISTS idx_table_metrics_composite
+ ON table_metrics(table_identifier, table_partition, metric_ts);
+CREATE INDEX IF NOT EXISTS idx_job_metrics_identifier_metric_ts
+ ON job_metrics(job_identifier, metric_ts);
+
+COMMENT ON TABLE table_metrics IS 'optimizer table metrics';
+COMMENT ON TABLE job_metrics IS 'optimizer job metrics';
+COMMENT ON COLUMN table_metrics.id IS 'auto increment id';
+COMMENT ON COLUMN table_metrics.table_identifier IS 'normalized table
identifier';
+COMMENT ON COLUMN table_metrics.metric_name IS 'metric name';
+COMMENT ON COLUMN table_metrics.table_partition IS 'normalized partition
identifier';
+COMMENT ON COLUMN table_metrics.metric_ts IS 'metric timestamp in epoch
seconds';
+COMMENT ON COLUMN table_metrics.metric_value IS 'metric value payload';
+COMMENT ON COLUMN job_metrics.id IS 'auto increment id';
+COMMENT ON COLUMN job_metrics.job_identifier IS 'normalized job identifier';
+COMMENT ON COLUMN job_metrics.metric_name IS 'metric name';
+COMMENT ON COLUMN job_metrics.metric_ts IS 'metric timestamp in epoch seconds';
+COMMENT ON COLUMN job_metrics.metric_value IS 'metric value payload';
diff --git a/scripts/postgresql/upgrade-1.1.0-to-1.2.0-postgresql.sql
b/scripts/postgresql/upgrade-1.1.0-to-1.2.0-postgresql.sql
index 7aa6bda401..e75d75735a 100644
--- a/scripts/postgresql/upgrade-1.1.0-to-1.2.0-postgresql.sql
+++ b/scripts/postgresql/upgrade-1.1.0-to-1.2.0-postgresql.sql
@@ -125,3 +125,42 @@ COMMENT ON COLUMN partition_statistic_meta.statistic_value
IS 'statistic value a
COMMENT ON COLUMN partition_statistic_meta.audit_info IS 'audit information as
JSON';
COMMENT ON COLUMN partition_statistic_meta.created_at IS 'creation timestamp
in milliseconds';
COMMENT ON COLUMN partition_statistic_meta.updated_at IS 'last update
timestamp in milliseconds';
+
+-- Add optimizer metrics storage tables
+CREATE TABLE IF NOT EXISTS table_metrics (
+ id BIGSERIAL PRIMARY KEY,
+ table_identifier VARCHAR(1024) NOT NULL,
+ metric_name VARCHAR(1024) NOT NULL,
+ table_partition VARCHAR(1024),
+ metric_ts BIGINT NOT NULL,
+ metric_value VARCHAR(1024) NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS job_metrics (
+ id BIGSERIAL PRIMARY KEY,
+ job_identifier VARCHAR(1024) NOT NULL,
+ metric_name VARCHAR(1024) NOT NULL,
+ metric_ts BIGINT NOT NULL,
+ metric_value VARCHAR(1024) NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_table_metrics_metric_ts ON
table_metrics(metric_ts);
+CREATE INDEX IF NOT EXISTS idx_job_metrics_metric_ts ON job_metrics(metric_ts);
+CREATE INDEX IF NOT EXISTS idx_table_metrics_composite
+ ON table_metrics(table_identifier, table_partition, metric_ts);
+CREATE INDEX IF NOT EXISTS idx_job_metrics_identifier_metric_ts
+ ON job_metrics(job_identifier, metric_ts);
+
+COMMENT ON TABLE table_metrics IS 'optimizer table metrics';
+COMMENT ON TABLE job_metrics IS 'optimizer job metrics';
+COMMENT ON COLUMN table_metrics.id IS 'auto increment id';
+COMMENT ON COLUMN table_metrics.table_identifier IS 'normalized table
identifier';
+COMMENT ON COLUMN table_metrics.metric_name IS 'metric name';
+COMMENT ON COLUMN table_metrics.table_partition IS 'normalized partition
identifier';
+COMMENT ON COLUMN table_metrics.metric_ts IS 'metric timestamp in epoch
seconds';
+COMMENT ON COLUMN table_metrics.metric_value IS 'metric value payload';
+COMMENT ON COLUMN job_metrics.id IS 'auto increment id';
+COMMENT ON COLUMN job_metrics.job_identifier IS 'normalized job identifier';
+COMMENT ON COLUMN job_metrics.metric_name IS 'metric name';
+COMMENT ON COLUMN job_metrics.metric_ts IS 'metric timestamp in epoch seconds';
+COMMENT ON COLUMN job_metrics.metric_value IS 'metric value payload';
