jerryshao commented on code in PR #14281: URL: https://github.com/apache/gravitino/pull/14281#discussion_r4409600516
########## core/src/main/java/org/apache/gravitino/meta/DomainEntity.java: ########## @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.meta; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import javax.annotation.Nullable; +import lombok.AccessLevel; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.Accessors; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.Audit; +import org.apache.gravitino.Auditable; +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityFieldLimits; +import org.apache.gravitino.Field; +import org.apache.gravitino.HasIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.domain.Domain; +import org.apache.gravitino.tag.TagValue; + +/** + * The stored form of a {@link Domain}. It holds the domain's data only; the operations on {@link + * Domain} are served by the domain manager. + * + * <p>The backing tag is stored with the domain as the ID of the tag and an optional value; the tag + * name is read from the tag when the domain is read. If the tag is deleted, the ID stays on the + * domain but has no name, so the domain has no backing tag. The parent is not stored with the + * domain: it comes from the sub-domain relation and is set on the entity when it is read. + */ +@Getter +@Accessors(fluent = true) +@EqualsAndHashCode +@ToString +public class DomainEntity implements Entity, Auditable, HasIdentifier { + + /** The unique id of the domain entity. */ + public static final Field ID = + Field.required("id", Long.class, "The unique id of the domain entity."); + + /** The name of the domain entity. */ + public static final Field NAME = + Field.required("name", "The name of the domain entity.", EntityFieldLimits.MAX_NAME_LENGTH); + + /** The comment of the domain entity. */ + public static final Field COMMENT = + Field.optional( + "comment", "The comment of the domain entity.", EntityFieldLimits.MAX_COMMENT_LENGTH); + + /** The properties of the domain entity. */ + public static final Field PROPERTIES = + Field.optional("properties", Map.class, "The properties of the domain entity."); + + /** The status of the domain entity. */ + public static final Field STATUS = + Field.required("status", Domain.Status.class, "The status of the domain entity."); + + /** The ID of the backing tag of the domain entity. */ + public static final Field BACKING_TAG_ID = + Field.optional( + "backing_tag_id", Long.class, "The ID of the backing tag of the domain entity."); + + /** The value of the backing tag of the domain entity. */ + public static final Field BACKING_TAG_VALUE = + Field.optional( + "backing_tag_value", + "The value of the backing tag of the domain entity.", + EntityFieldLimits.MAX_TAG_VALUE_LENGTH); + + /** The audit details of the domain entity. */ + public static final Field AUDIT_INFO = + Field.required("audit_info", Audit.class, "The audit details of the domain entity."); + + private final Long id; + private final String name; + private final Namespace namespace; + @Nullable private final String comment; + private final Map<String, String> properties; + private final Domain.Status status; + + @Getter(AccessLevel.NONE) + @Nullable + private final String parent; + + /** The ID of the backing tag, or null if the domain has none. */ + @Nullable private final Long backingTagId; + + /** The name of the backing tag, or null if the domain has none or the tag is deleted. */ + @Getter(AccessLevel.NONE) + @Nullable + private final String backingTagName; + + /** The value of the backing tag, or null if the tag is bound without a value. */ + @Nullable private final String backingTagValue; + + private final Audit auditInfo; + + @lombok.Builder(setterPrefix = "with", toBuilder = true) + private DomainEntity( + Long id, + String name, + Namespace namespace, + @Nullable String comment, + Map<String, String> properties, + Domain.Status status, + @Nullable String parent, + @Nullable Long backingTagId, + @Nullable String backingTagName, + @Nullable String backingTagValue, + Audit auditInfo) { + this.id = id; + this.name = name; + this.namespace = namespace; + this.comment = comment; + this.properties = properties; + this.status = status; + this.parent = parent; + this.backingTagId = backingTagId; + this.backingTagName = backingTagName; + this.backingTagValue = backingTagValue; + this.auditInfo = auditInfo; + validate(); + Preconditions.checkArgument( + backingTagId != null || (backingTagName == null && backingTagValue == null), + "The backing tag name and value of domain %s need a backing tag ID", + name); Review Comment: [Important] Unbinding a backing tag through `toBuilder()` fails unless the caller also clears the derived name. `backingTagName` is a derived field: it is not a column (`DomainPO` has no `withBackingTagName` setter), its getter is suppressed with `@Getter(AccessLevel.NONE)`, and reads refill it from the live tag via `BACKING_TAG_JOIN`. But it *is* a `toBuilder()` parameter, and this precondition rejects a non-null name whenever the ID is null. So the natural "remove the backing tag" alter entity.toBuilder().withBackingTagId(null).withBackingTagValue(null).build() throws `IllegalArgumentException` with the message "The backing tag name and value of domain X need a backing tag ID" on any entity that was read back with a live tag, even though the caller did clear the value. The domain manager (#13884) is the first caller that will hit this. Suggest making the name follow the ID instead of constraining it: either normalize in the constructor (`this.backingTagName = backingTagId == null ? null : backingTagName;`) or drop `backingTagName` from the precondition and keep only the `backingTagValue` term. Verified by: read `DomainEntity.java:119-147` (the builder constructor and both preconditions) and confirmed `backingTagName` is absent from `DomainPO`'s builder constructor (`DomainPO.java:68-81`) while `DomainPO.fromDomainPO` sets it from the joined column (`DomainPO.java:140-147`). The PR's own test is the evidence it bites: `TestDomainMetaService.java:503-511` is the only place in the repo that calls `withBackingTagName(null)`, and it does so solely to make this unbind build succeed (`grep -rn withBackingTagName core/src` returns that one hit). ########## scripts/mysql/schema-2.0.0-mysql.sql: ########## @@ -752,3 +752,21 @@ CREATE TABLE IF NOT EXISTS `table_snapshot_metrics` ( UNIQUE KEY `uk_tid_pid_sid` (`table_id`, `policy_id`, `snapshot_id`), KEY `idx_tsm_created_at` (`created_at`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'table snapshot metrics metadata'; + +CREATE TABLE IF NOT EXISTS `domain_meta` ( + `domain_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'domain id', + `domain_name` VARCHAR(128) NOT NULL COMMENT 'domain name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `domain_comment` VARCHAR(256) DEFAULT '' COMMENT 'domain comment', + `domain_status` VARCHAR(16) NOT NULL DEFAULT 'DRAFT' COMMENT 'domain status, DRAFT or PUBLISHED', + `backing_tag_id` BIGINT(20) UNSIGNED DEFAULT NULL COMMENT 'id of the backing tag, NULL if the domain has none', + `backing_tag_value` VARCHAR(256) DEFAULT NULL COMMENT 'value of the backing tag, NULL binds the tag without a value', Review Comment: [Question] "No tag value" is NULL here but the empty string in `tag_relation_meta` - the reverse lookup will have to translate. This column stores an unvalued binding as NULL, and `DomainEntity` enforces that (a blank value is rejected, `DomainEntity.java:149-152`). The tag assignment table spells the same state differently: `tag_relation_meta.tag_value` is `VARCHAR(256) NOT NULL DEFAULT '' COMMENT 'tag assignment value, empty string means no value'` (`scripts/mysql/schema-2.0.0-mysql.sql:304`, with `KEY idx_tid_value (tag_id, tag_value)` at :313). The stated purpose of `domain_meta_idx_backing_tag` is the reverse lookup from a tag assignment to the domains it backs. Across the two spellings that lookup cannot be a plain equality join: `backing_tag_value = ''` never matches an unvalued domain, and `= NULL` never matches anything, so the caller needs an explicit `tag_value = '' -> IS NULL` mapping, and a `WHERE (tag_id, tag_value) IN (...)` form over mixed rows will silently drop the unvalued ones. Is the divergence deliberate (NULL being the more honest encoding for a new table), with the translation planned for the sub-domain/manager subtasks? If so a line in the column comment or in `DomainPO`'s class Javadoc pointing at it would save the next author the discovery. Otherwise matching `tag_relation_meta` and using `''` would let the reverse lookup join directly. Verified by: read the new table definition in all four script pairs and compared with the `tag_relation_meta` definition at `scripts/mysql/schema-2.0.0-mysql.sql:295-314`; read `TagValue.of`/`TagValue.noValue` (`api/.../tag/TagValue.java:49-62`), which also rejects a blank value, so the empty string is unreachable from the API side and only exists at the storage layer. ########## core/src/main/java/org/apache/gravitino/storage/relational/po/DomainPO.java: ########## @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.po; + +import static org.apache.gravitino.storage.relational.utils.POConverters.DEFAULT_DELETED_AT; +import static org.apache.gravitino.storage.relational.utils.POConverters.INIT_VERSION; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.base.Preconditions; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.Accessors; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.domain.Domain; +import org.apache.gravitino.json.JsonUtils; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.DomainEntity; + +/** + * The persistent object of a row in {@code domain_meta}. The backing tag name is not a column of + * the row: reads that join the live tag fill it, and it is null when the domain has no backing tag + * or the tag is deleted. + */ +@Getter +@Accessors(fluent = true) +@EqualsAndHashCode +@ToString +public class DomainPO { + + private Long domainId; + private String domainName; + private Long metalakeId; + private String comment; + private String status; + private Long backingTagId; + private String backingTagName; + private String backingTagValue; + private String properties; + private String auditInfo; + private Long currentVersion; + private Long lastVersion; + private Long deletedAt; + + /** Creates an empty domain persistent object, used by MyBatis to map result rows. */ + public DomainPO() {} + + @lombok.Builder(setterPrefix = "with") + private DomainPO( + Long domainId, + String domainName, + Long metalakeId, + String comment, + String status, + Long backingTagId, + String backingTagValue, + String properties, + String auditInfo, + Long currentVersion, + Long lastVersion, + Long deletedAt) { + Preconditions.checkArgument(domainId != null, "domainId cannot be null"); + Preconditions.checkArgument(StringUtils.isNotBlank(domainName), "domainName cannot be blank"); + Preconditions.checkArgument(metalakeId != null, "metalakeId cannot be null"); + Preconditions.checkArgument(StringUtils.isNotBlank(status), "status cannot be blank"); + Preconditions.checkArgument( + backingTagId != null || backingTagValue == null, + "backingTagValue cannot be set without backingTagId"); + Preconditions.checkArgument(StringUtils.isNotBlank(auditInfo), "auditInfo cannot be blank"); + Preconditions.checkArgument(currentVersion != null, "currentVersion cannot be null"); + Preconditions.checkArgument(lastVersion != null, "lastVersion cannot be null"); + Preconditions.checkArgument(deletedAt != null, "deletedAt cannot be null"); + + this.domainId = domainId; + this.domainName = domainName; + this.metalakeId = metalakeId; + this.comment = comment; + this.status = status; + this.backingTagId = backingTagId; + this.backingTagValue = backingTagValue; + this.properties = properties; + this.auditInfo = auditInfo; + this.currentVersion = currentVersion; + this.lastVersion = lastVersion; + this.deletedAt = deletedAt; + } + + /** + * Creates the first version of the persistent object of a new domain. + * + * @param domainEntity The domain entity. + * @param metalakeId The ID of the metalake of the domain. + * @return The domain persistent object. + */ + public static DomainPO initializeDomainPO(DomainEntity domainEntity, Long metalakeId) { + return toDomainPO(domainEntity, domainEntity.id(), metalakeId, INIT_VERSION); + } + + /** + * Creates the next version of the persistent object of a domain from its updated entity. + * + * @param oldDomainPO The domain persistent object observed by the caller. + * @param newDomainEntity The updated domain entity. + * @return The domain persistent object with the next version. + */ + public static DomainPO updateDomainPO(DomainPO oldDomainPO, DomainEntity newDomainEntity) { + Long nextVersion = oldDomainPO.currentVersion() + 1; + return toDomainPO( + newDomainEntity, oldDomainPO.domainId(), oldDomainPO.metalakeId(), nextVersion); + } + + /** + * Converts a domain persistent object to a domain entity. The parent and the inherited source are Review Comment: [Nit] Javadoc mentions an "inherited source" that `DomainEntity` does not have. The sentence promises two unset things, but the entity has only `parent` among them: its fields are id, name, namespace, comment, properties, status, parent, backingTagId, backingTagName, backingTagValue, auditInfo. "The inherited source" looks like a leftover from an earlier revision of the design. Suggest trimming to the parent (and the backing tag name, which this method does leave null for the caller to fill from the join). Verified by: read all field declarations in `DomainEntity.java:93-117` and the builder constructor at :119-131; no inherited-source field or accessor exists, and `grep -rn "inherited" core/src/main/java/org/apache/gravitino/meta/DomainEntity.java` returns nothing. ########## core/src/main/java/org/apache/gravitino/storage/relational/service/DomainMetaService.java: ########## @@ -0,0 +1,439 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.service; + +import static org.apache.gravitino.metrics.source.MetricsSource.GRAVITINO_RELATIONAL_STORE_METRIC_NAME; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityAlreadyExistsException; +import org.apache.gravitino.HasIdentifier; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.exceptions.NoSuchEntityException; +import org.apache.gravitino.meta.DomainEntity; +import org.apache.gravitino.metrics.Monitored; +import org.apache.gravitino.storage.relational.mapper.DomainMetaMapper; +import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper; +import org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper; +import org.apache.gravitino.storage.relational.mapper.SecurableObjectMapper; +import org.apache.gravitino.storage.relational.po.DomainPO; +import org.apache.gravitino.storage.relational.po.MetalakePO; +import org.apache.gravitino.storage.relational.utils.ExceptionUtils; +import org.apache.gravitino.storage.relational.utils.SessionUtils; + +/** The service that stores domains in the {@code domain_meta} table. */ +public class DomainMetaService { + + private static final DomainMetaService INSTANCE = new DomainMetaService(); + + private DomainMetaService() {} + + /** + * Returns the singleton service instance. + * + * @return The domain meta service. + */ + public static DomainMetaService getInstance() { + return INSTANCE; + } + + /** + * Lists the domains of a metalake. + * + * @param ns The domain namespace of the metalake. + * @return The domains of the metalake. + */ + @Monitored( + metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME, + baseMetricName = "listDomainsByNamespace") + public List<DomainEntity> listDomainsByNamespace(Namespace ns) { + String metalakeName = ns.level(0); + List<DomainPO> domainPOs = + SessionUtils.getWithoutCommit( + DomainMetaMapper.class, mapper -> mapper.listDomainPOsByMetalake(metalakeName)); + return DomainPO.fromDomainPOs(domainPOs, ns); + } + + /** + * Gets a domain by its identifier. + * + * @param ident The domain identifier. + * @return The domain. + * @throws NoSuchEntityException If the domain does not exist. + */ + @Monitored( + metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME, + baseMetricName = "getDomainByIdentifier") + public DomainEntity getDomainByIdentifier(NameIdentifier ident) { + String metalakeName = ident.namespace().level(0); + DomainPO domainPO = getDomainPOByMetalakeAndName(metalakeName, ident.name()); + return DomainPO.fromDomainPO(domainPO, ident.namespace()); + } + + /** + * Gets the domains with the given identifiers. Identifiers of domains that do not exist are + * skipped. + * + * @param identifiers The domain identifiers, all in the same metalake. + * @return The domains that exist. + */ + @Monitored( + metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME, + baseMetricName = "batchGetDomainByIdentifier") + public List<DomainEntity> batchGetDomainByIdentifier(List<NameIdentifier> identifiers) { + NameIdentifier firstIdent = identifiers.get(0); Review Comment: [Nit] `identifiers.get(0)` throws on an empty list. `batchGetDomainByIdentifier` dereferences element 0 before checking the list, so an empty argument gives `IndexOutOfBoundsException` instead of an empty result. It is not reachable through the cache path today - `RelationalEntityStore.batchGet` can hand `backend.batchGet` an empty list after filtering cache hits, but `DOMAIN` is deliberately not in `BaseEntityCache.CACHEABLE_TYPES`, so nothing is filtered - and `TagMetaService.batchGetTagByIdentifier` has the same shape, so this is a pattern rather than a regression. Still, a two-line `if (identifiers.isEmpty()) return Collections.emptyList();` makes the new entry point safe for the manager subtask without waiting on the other services. Verified by: read `DomainMetaService.java:107-118`, compared with `TagMetaService.java:958-962`, traced the empty-list path through `RelationalEntityStore.batchGet` (`RelationalEntityStore.java:283-299`) and confirmed `DOMAIN` is absent from the `CACHEABLE_TYPES` allowlist (`BaseEntityCache.java:55-64`). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
