lasdf1234 commented on code in PR #13774:
URL: https://github.com/apache/gravitino/pull/13774#discussion_r4259887575
##########
scripts/mysql/schema-2.0.0-mysql.sql:
##########
@@ -668,23 +668,73 @@ CREATE TABLE IF NOT EXISTS `iceberg_cleanup_job` (
KEY `idx_object` (`catalog_id`, `namespace`(255), `table_name`(128), `state`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'async
Iceberg table cleanup jobs';
+CREATE TABLE IF NOT EXISTS `scheduled_expand_tasks` (
+ `task_name` VARCHAR(100) NOT NULL COMMENT 'task name',
+ `task_instance` VARCHAR(100) NOT NULL COMMENT 'task instance',
+ `task_data` BLOB COMMENT 'serialized task data',
+ `execution_time` TIMESTAMP(6) NOT NULL COMMENT 'next execution time',
+ `picked` BOOLEAN NOT NULL COMMENT 'whether the task is currently picked',
+ `picked_by` VARCHAR(50) COMMENT 'scheduler instance that picked the task',
+ `last_heartbeat` TIMESTAMP(6) NULL COMMENT 'last heartbeat from the
picker',
+ `version` BIGINT NOT NULL COMMENT 'optimistic lock version',
+ `consecutive_failures` INT COMMENT 'consecutive failure count',
+ `last_success` TIMESTAMP(6) NULL COMMENT 'last successful execution time',
+ `last_failure` TIMESTAMP(6) NULL COMMENT 'last failed execution time',
+ PRIMARY KEY (`task_name`, `task_instance`),
+ KEY `execution_time_idx` (`execution_time`),
+ KEY `last_heartbeat_idx` (`last_heartbeat`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'table
maintenance expand schedule metadata';
+
+CREATE TABLE IF NOT EXISTS `scheduled_cron_tasks` (
+ `task_name` VARCHAR(100) NOT NULL COMMENT 'task name',
+ `task_instance` VARCHAR(100) NOT NULL COMMENT 'task instance',
+ `task_data` BLOB COMMENT 'serialized task data',
+ `execution_time` TIMESTAMP(6) NOT NULL COMMENT 'next execution time',
+ `picked` BOOLEAN NOT NULL COMMENT 'whether the task is currently picked',
+ `picked_by` VARCHAR(50) COMMENT 'scheduler instance that picked the task',
+ `last_heartbeat` TIMESTAMP(6) NULL COMMENT 'last heartbeat from the
picker',
+ `version` BIGINT NOT NULL COMMENT 'optimistic lock version',
+ `consecutive_failures` INT COMMENT 'consecutive failure count',
+ `last_success` TIMESTAMP(6) NULL COMMENT 'last successful execution time',
+ `last_failure` TIMESTAMP(6) NULL COMMENT 'last failed execution time',
+ PRIMARY KEY (`task_name`, `task_instance`),
+ KEY `execution_time_idx` (`execution_time`),
+ KEY `last_heartbeat_idx` (`last_heartbeat`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'table
maintenance cron schedule metadata';
+
+CREATE TABLE IF NOT EXISTS `scheduled_commit_tasks` (
+ `task_name` VARCHAR(100) NOT NULL COMMENT 'task name',
+ `task_instance` VARCHAR(100) NOT NULL COMMENT 'task instance',
+ `task_data` BLOB COMMENT 'serialized task data',
+ `execution_time` TIMESTAMP(6) NOT NULL COMMENT 'next execution time',
+ `picked` BOOLEAN NOT NULL COMMENT 'whether the task is currently picked',
+ `picked_by` VARCHAR(50) COMMENT 'scheduler instance that picked the task',
+ `last_heartbeat` TIMESTAMP(6) NULL COMMENT 'last heartbeat from the
picker',
+ `version` BIGINT NOT NULL COMMENT 'optimistic lock version',
+ `consecutive_failures` INT COMMENT 'consecutive failure count',
+ `last_success` TIMESTAMP(6) NULL COMMENT 'last successful execution time',
+ `last_failure` TIMESTAMP(6) NULL COMMENT 'last failed execution time',
+ PRIMARY KEY (`task_name`, `task_instance`),
+ KEY `execution_time_idx` (`execution_time`),
+ KEY `last_heartbeat_idx` (`last_heartbeat`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'table
maintenance commit schedule metadata';
+
CREATE TABLE IF NOT EXISTS `table_maintenance_action` (
`id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment
id',
- `job_id` BIGINT(20) UNSIGNED NULL COMMENT 'optional job run id mapped to
job_run_meta.job_run_id, null until bound after runJob',
+ `job_id` BIGINT(20) UNSIGNED NULL COMMENT 'job run id, maps to
job_run_meta.job_run_id',
`table_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'table id from table_meta',
`policy_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'policy id from
policy_meta',
`before_snapshot_id` BIGINT(20) NULL COMMENT 'Iceberg snapshot id before
maintenance, null until sampled',
`after_snapshot_id` BIGINT(20) NULL COMMENT 'Iceberg snapshot id after
maintenance, null while pending',
- `finished_at` BIGINT(20) UNSIGNED NULL COMMENT 'finish timestamp in
milliseconds, null while in flight',
+ `finished_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'finish
timestamp in milliseconds, 0 while in flight',
`created_at` BIGINT(20) UNSIGNED NOT NULL COMMENT 'creation timestamp in
milliseconds',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_tma_job_id` (`job_id`),
- KEY `idx_tma_table_policy_finished` (`table_id`, `policy_id`,
`finished_at`),
+ UNIQUE KEY `uk_tma_table_policy_finished` (`table_id`, `policy_id`,
`finished_at`),
Review Comment:
Thanks for the clarification — we understand the collision does not need
concurrent finishers; sequential same-millisecond (or clock-rollback)
`markFinished` can hit the unique key.
We are **keeping** `UNIQUE(table_id, policy_id, finished_at)` for this PR
and not switching to a partial / generated-column soft-slot-only unique.
**Why:** the soft in-flight slot is modeled the same way as Gravitino’s
existing soft-delete keys (`UNIQUE(..., deleted_at)` with `0` = live). That
single compound unique is what enforces “at most one `finished_at = 0` per
`(table_id, policy_id)`” without an extra occupancy table or MySQL-specific
generated columns. We rely on that pattern for acquire / insert races.
A history-non-unique design (PG `UNIQUE (...) WHERE finished_at = 0`, or
MySQL generated columns that are NULL when finished) would address the same-ms
finish case, but it diverges from the repo’s established soft-slot idiom and
adds dialect-specific DDL we do not want to introduce here.
We documented the soft-slot intent in SQL / Javadoc. If same-ms finish
collisions show up in practice, we can revisit `markFinished` (e.g.
bump-on-conflict) or a conditional unique later — without changing the
soft-slot model in this PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
