jerqi commented on code in PR #7734: URL: https://github.com/apache/gravitino/pull/7734#discussion_r2272268914
########## core/src/main/java/org/apache/gravitino/Relation.java: ########## @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino; + +import java.util.Objects; + +/** Represents a relation between two entities in the metadata store. */ +public class Relation { + private final Vertex sourceVertex; + private final Vertex destVertex; + + /** + * Creates a new Relation instance. + * + * @param sourceIdent the identifier of the source entity + * @param sourceType the type of the source entity + * @param destIdent the identifier of the destination entity + * @param destType the type of the destination entity + */ + public Relation( + NameIdentifier sourceIdent, + Entity.EntityType sourceType, + NameIdentifier destIdent, + Entity.EntityType destType) { + this.sourceVertex = new Vertex(sourceIdent, sourceType); + this.destVertex = new Vertex(destIdent, destType); + } + + /** + * Returns the source vertex of the relation. + * + * @return the source vertex + */ + public Vertex getSourceVertex() { + return sourceVertex; + } + + /** + * Returns the destination vertex of the relation. + * + * @return the destination vertex + */ + public Vertex getDestVertex() { + return destVertex; + } + + /** + * Returns the identifier of the source entity. + * + * @return the source identifier + */ + public NameIdentifier getSourceIdent() { + return sourceVertex.getIdentifier(); + } + + /** + * Returns the type of the source entity. + * + * @return the source entity type + */ + public Entity.EntityType getSourceType() { + return sourceVertex.getType(); + } + + /** + * Returns the identifier of the destination entity. + * + * @return the destination identifier + */ + public NameIdentifier getDestIdent() { + return destVertex.getIdentifier(); + } + + /** + * Returns the type of the destination entity. + * + * @return the destination entity type + */ + public Entity.EntityType getDestType() { + return destVertex.getType(); + } + + /** + * Represents a vertex in the relation graph, which consists of an identifier and an entity type. + */ + public static class Vertex { Review Comment: Removed the class Vertex. ########## core/src/main/java/org/apache/gravitino/stats/StatisticManager.java: ########## @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.stats; + +import com.google.common.collect.Lists; +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityStore; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.Relation; +import org.apache.gravitino.SupportsRelationOperations; +import org.apache.gravitino.exceptions.NoSuchEntityException; +import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; +import org.apache.gravitino.exceptions.UnmodifiableStatisticException; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.StatisticEntity; +import org.apache.gravitino.storage.IdGenerator; +import org.apache.gravitino.utils.MetadataObjectUtil; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.PrincipalUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StatisticManager { + + private static final Logger LOG = LoggerFactory.getLogger(StatisticManager.class); + + private final EntityStore store; + + private final IdGenerator idGenerator; + + public StatisticManager(EntityStore store, IdGenerator idGenerator) { + this.store = store; + this.idGenerator = idGenerator; + } + + public List<Statistic> listStatistics(String metalake, MetadataObject metadataObject) { + try { + // TODO: we will add lock when we support Iceberg, Hive statistics + NameIdentifier identifier = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType type = MetadataObjectUtil.toEntityType(metadataObject); + return store.relationOperations() + .listEntitiesByRelation( + SupportsRelationOperations.Type.METADATA_OBJECT_STAT_REL, identifier, type) + .stream() + .map( + entity -> { + StatisticEntity statisticEntity = (StatisticEntity) entity; + String name = statisticEntity.name(); + StatisticValue<?> value = statisticEntity.value(); + return new CustomStatistic(name, value); + }) + .collect(Collectors.toList()); + } catch (NoSuchEntityException nse) { + LOG.warn( + "Failed to list statistics for metadata object {}: {}", + metadataObject.fullName(), + nse.getMessage()); + throw new NoSuchMetadataObjectException( + "The metadata object of %s isn't found", metadataObject.fullName()); Review Comment: Done. ########## core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java: ########## @@ -569,6 +594,76 @@ public <E extends Entity & HasIdentifier> List<E> updateEntityRelations( } } + @Override + public <E extends Entity & HasIdentifier> void insertEntitiesAndRelations( + Type relType, List<E> entities, List<Relation> relations, boolean overwrite) + throws IOException { + switch (relType) { + case METADATA_OBJECT_STAT_REL: + if (!overwrite) { + throw new IllegalArgumentException( + "The overwrite must be true for metadata object stats relation"); + } Review Comment: Done. ########## core/src/main/java/org/apache/gravitino/stats/StatisticManager.java: ########## @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.stats; + +import com.google.common.collect.Lists; +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityStore; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.Relation; +import org.apache.gravitino.SupportsRelationOperations; +import org.apache.gravitino.exceptions.NoSuchEntityException; +import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; +import org.apache.gravitino.exceptions.UnmodifiableStatisticException; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.StatisticEntity; +import org.apache.gravitino.storage.IdGenerator; +import org.apache.gravitino.utils.MetadataObjectUtil; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.PrincipalUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StatisticManager { + + private static final Logger LOG = LoggerFactory.getLogger(StatisticManager.class); + + private final EntityStore store; + + private final IdGenerator idGenerator; + + public StatisticManager(EntityStore store, IdGenerator idGenerator) { + this.store = store; + this.idGenerator = idGenerator; + } + + public List<Statistic> listStatistics(String metalake, MetadataObject metadataObject) { + try { + // TODO: we will add lock when we support Iceberg, Hive statistics Review Comment: I added more comments to explain. Because the operations are only related to entity store now. We don't need to lock the metadata object. ########## core/src/main/java/org/apache/gravitino/stats/StatisticManager.java: ########## @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.stats; + +import com.google.common.collect.Lists; +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityStore; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.Relation; +import org.apache.gravitino.SupportsRelationOperations; +import org.apache.gravitino.exceptions.NoSuchEntityException; +import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; +import org.apache.gravitino.exceptions.UnmodifiableStatisticException; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.StatisticEntity; +import org.apache.gravitino.storage.IdGenerator; +import org.apache.gravitino.utils.MetadataObjectUtil; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.PrincipalUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StatisticManager { + + private static final Logger LOG = LoggerFactory.getLogger(StatisticManager.class); + + private final EntityStore store; + + private final IdGenerator idGenerator; + + public StatisticManager(EntityStore store, IdGenerator idGenerator) { + this.store = store; + this.idGenerator = idGenerator; + } + + public List<Statistic> listStatistics(String metalake, MetadataObject metadataObject) { + try { + // TODO: we will add lock when we support Iceberg, Hive statistics + NameIdentifier identifier = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType type = MetadataObjectUtil.toEntityType(metadataObject); + return store.relationOperations() + .listEntitiesByRelation( + SupportsRelationOperations.Type.METADATA_OBJECT_STAT_REL, identifier, type) + .stream() + .map( + entity -> { + StatisticEntity statisticEntity = (StatisticEntity) entity; + String name = statisticEntity.name(); + StatisticValue<?> value = statisticEntity.value(); + return new CustomStatistic(name, value); + }) + .collect(Collectors.toList()); + } catch (NoSuchEntityException nse) { + LOG.warn( + "Failed to list statistics for metadata object {}: {}", + metadataObject.fullName(), + nse.getMessage()); + throw new NoSuchMetadataObjectException( + "The metadata object of %s isn't found", metadataObject.fullName()); + } catch (IOException ioe) { + LOG.error( + "Failed to list statistics for metadata object {}: {}", + metadataObject.fullName(), + ioe.getMessage()); + throw new RuntimeException(ioe); + } + } + + public List<Statistic> updateStatistics( + String metalake, MetadataObject metadataObject, Map<String, StatisticValue<?>> statistics) { + try { + // TODO: we will add lock when we support Iceberg, Hive statistics + NameIdentifier identifier = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType type = MetadataObjectUtil.toEntityType(metadataObject); + List<Relation> relations = Lists.newArrayList(); + List<StatisticEntity> statisticEntities = Lists.newArrayList(); + for (Map.Entry<String, StatisticValue<?>> entry : statistics.entrySet()) { + String name = entry.getKey(); + StatisticValue<?> value = entry.getValue(); + + StatisticEntity statistic = + StatisticEntity.builder() + .withId(idGenerator.nextId()) + .withName(name) + .withValue(value) + .withNamespace(Namespace.fromString(identifier.toString())) + .withAuditInfo( + AuditInfo.builder() + .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) + .withCreateTime(Instant.now()) + .withLastModifier(PrincipalUtils.getCurrentPrincipal().getName()) + .withLastModifiedTime(Instant.now()) + .build()) + .build(); + statisticEntities.add(statistic); + relations.add( + new Relation( + identifier, type, statistic.nameIdentifier(), Entity.EntityType.STATISTIC)); + } + store + .relationOperations() + .insertEntitiesAndRelations( + SupportsRelationOperations.Type.METADATA_OBJECT_STAT_REL, + statisticEntities, + relations, + true); + return statistics.entrySet().stream() + .map(entry -> new CustomStatistic(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + } catch (NoSuchEntityException nse) { + LOG.warn( + "Failed to update statistics for metadata object {}: {}", + metadataObject.fullName(), + nse.getMessage()); Review Comment: Done. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
