jerryshao commented on code in PR #7734: URL: https://github.com/apache/gravitino/pull/7734#discussion_r2272075875
########## core/src/main/java/org/apache/gravitino/stats/StatisticManager.java: ########## @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.stats; + +import com.google.common.collect.Lists; +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityStore; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.Relation; +import org.apache.gravitino.SupportsRelationOperations; +import org.apache.gravitino.exceptions.NoSuchEntityException; +import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; +import org.apache.gravitino.exceptions.UnmodifiableStatisticException; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.StatisticEntity; +import org.apache.gravitino.storage.IdGenerator; +import org.apache.gravitino.utils.MetadataObjectUtil; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.PrincipalUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StatisticManager { + + private static final Logger LOG = LoggerFactory.getLogger(StatisticManager.class); + + private final EntityStore store; + + private final IdGenerator idGenerator; + + public StatisticManager(EntityStore store, IdGenerator idGenerator) { + this.store = store; + this.idGenerator = idGenerator; + } + + public List<Statistic> listStatistics(String metalake, MetadataObject metadataObject) { + try { + // TODO: we will add lock when we support Iceberg, Hive statistics Review Comment: Can you explain this comment? ########## core/src/main/java/org/apache/gravitino/Relation.java: ########## @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino; + +import java.util.Objects; + +/** Represents a relation between two entities in the metadata store. */ +public class Relation { + private final Vertex sourceVertex; + private final Vertex destVertex; + + /** + * Creates a new Relation instance. + * + * @param sourceIdent the identifier of the source entity + * @param sourceType the type of the source entity + * @param destIdent the identifier of the destination entity + * @param destType the type of the destination entity + */ + public Relation( + NameIdentifier sourceIdent, + Entity.EntityType sourceType, + NameIdentifier destIdent, + Entity.EntityType destType) { + this.sourceVertex = new Vertex(sourceIdent, sourceType); + this.destVertex = new Vertex(destIdent, destType); + } + + /** + * Returns the source vertex of the relation. + * + * @return the source vertex + */ + public Vertex getSourceVertex() { + return sourceVertex; + } + + /** + * Returns the destination vertex of the relation. + * + * @return the destination vertex + */ + public Vertex getDestVertex() { + return destVertex; + } + + /** + * Returns the identifier of the source entity. + * + * @return the source identifier + */ + public NameIdentifier getSourceIdent() { + return sourceVertex.getIdentifier(); + } + + /** + * Returns the type of the source entity. + * + * @return the source entity type + */ + public Entity.EntityType getSourceType() { + return sourceVertex.getType(); + } + + /** + * Returns the identifier of the destination entity. + * + * @return the destination identifier + */ + public NameIdentifier getDestIdent() { + return destVertex.getIdentifier(); + } + + /** + * Returns the type of the destination entity. + * + * @return the destination entity type + */ + public Entity.EntityType getDestType() { + return destVertex.getType(); + } + + /** + * Represents a vertex in the relation graph, which consists of an identifier and an entity type. + */ + public static class Vertex { Review Comment: We can use `Pair`, we don't have to bring in new concept for this. ########## core/src/main/java/org/apache/gravitino/SupportsRelationOperations.java: ########## @@ -148,4 +149,32 @@ default <E extends Entity & HasIdentifier> List<E> updateEntityRelations( throw new UnsupportedOperationException( "updateEntityRelations is not supported by this implementation"); } + + /** + * Inserts a list of entities and their associated relations into the storage. + * + * @param <E> The type of entities being inserted, which must extend Entity and implement + * HasIdentifier. + * @param relType The type of relation to be established between the entities. + * @param entities The list of entities to be inserted, which must implement the Entity and + * HasIdentifier + * @param relations The list of relations to be established between the entities. + * @param overwrite If true, existing relations will be overwritten; if false, existing relations + * will throw an exception. + * @throws IOException If an error occurs during the insertion process, such as a storage issue. + */ + <E extends Entity & HasIdentifier> void insertEntitiesAndRelations( + Type relType, List<E> entities, List<Relation> relations, boolean overwrite) + throws IOException; + + /** + * Deletes a list of relations of a specific type. + * + * @param relType The type of relation to be deleted. + * @param relations The list of relations to be deleted, which must implement the Relation + * interface. + * @return The number of relations deleted. + * @throws IOException If an error occurs during the deletion process, such as a storage issue. + */ + int deleteRelations(Type relType, List<Relation> relations) throws IOException; Review Comment: Do you have overlap with @mchades 's change, I rememebered that he also changed this part. ########## core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java: ########## @@ -569,6 +594,76 @@ public <E extends Entity & HasIdentifier> List<E> updateEntityRelations( } } + @Override + public <E extends Entity & HasIdentifier> void insertEntitiesAndRelations( + Type relType, List<E> entities, List<Relation> relations, boolean overwrite) + throws IOException { + switch (relType) { + case METADATA_OBJECT_STAT_REL: + if (!overwrite) { + throw new IllegalArgumentException( + "The overwrite must be true for metadata object stats relation"); + } Review Comment: Use "Preconditions". ########## core/src/main/java/org/apache/gravitino/stats/StatisticManager.java: ########## @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.stats; + +import com.google.common.collect.Lists; +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityStore; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.Relation; +import org.apache.gravitino.SupportsRelationOperations; +import org.apache.gravitino.exceptions.NoSuchEntityException; +import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; +import org.apache.gravitino.exceptions.UnmodifiableStatisticException; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.StatisticEntity; +import org.apache.gravitino.storage.IdGenerator; +import org.apache.gravitino.utils.MetadataObjectUtil; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.PrincipalUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StatisticManager { + + private static final Logger LOG = LoggerFactory.getLogger(StatisticManager.class); + + private final EntityStore store; + + private final IdGenerator idGenerator; + + public StatisticManager(EntityStore store, IdGenerator idGenerator) { + this.store = store; + this.idGenerator = idGenerator; + } + + public List<Statistic> listStatistics(String metalake, MetadataObject metadataObject) { + try { + // TODO: we will add lock when we support Iceberg, Hive statistics + NameIdentifier identifier = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType type = MetadataObjectUtil.toEntityType(metadataObject); + return store.relationOperations() + .listEntitiesByRelation( + SupportsRelationOperations.Type.METADATA_OBJECT_STAT_REL, identifier, type) + .stream() + .map( + entity -> { + StatisticEntity statisticEntity = (StatisticEntity) entity; + String name = statisticEntity.name(); + StatisticValue<?> value = statisticEntity.value(); + return new CustomStatistic(name, value); + }) + .collect(Collectors.toList()); + } catch (NoSuchEntityException nse) { + LOG.warn( + "Failed to list statistics for metadata object {}: {}", + metadataObject.fullName(), + nse.getMessage()); + throw new NoSuchMetadataObjectException( + "The metadata object of %s isn't found", metadataObject.fullName()); + } catch (IOException ioe) { + LOG.error( + "Failed to list statistics for metadata object {}: {}", + metadataObject.fullName(), + ioe.getMessage()); + throw new RuntimeException(ioe); + } + } + + public List<Statistic> updateStatistics( + String metalake, MetadataObject metadataObject, Map<String, StatisticValue<?>> statistics) { + try { + // TODO: we will add lock when we support Iceberg, Hive statistics + NameIdentifier identifier = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType type = MetadataObjectUtil.toEntityType(metadataObject); + List<Relation> relations = Lists.newArrayList(); + List<StatisticEntity> statisticEntities = Lists.newArrayList(); + for (Map.Entry<String, StatisticValue<?>> entry : statistics.entrySet()) { + String name = entry.getKey(); + StatisticValue<?> value = entry.getValue(); + + StatisticEntity statistic = + StatisticEntity.builder() + .withId(idGenerator.nextId()) + .withName(name) + .withValue(value) + .withNamespace(Namespace.fromString(identifier.toString())) + .withAuditInfo( + AuditInfo.builder() + .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) + .withCreateTime(Instant.now()) + .withLastModifier(PrincipalUtils.getCurrentPrincipal().getName()) + .withLastModifiedTime(Instant.now()) + .build()) + .build(); + statisticEntities.add(statistic); + relations.add( + new Relation( + identifier, type, statistic.nameIdentifier(), Entity.EntityType.STATISTIC)); + } + store + .relationOperations() + .insertEntitiesAndRelations( + SupportsRelationOperations.Type.METADATA_OBJECT_STAT_REL, + statisticEntities, + relations, + true); + return statistics.entrySet().stream() + .map(entry -> new CustomStatistic(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + } catch (NoSuchEntityException nse) { + LOG.warn( + "Failed to update statistics for metadata object {}: {}", + metadataObject.fullName(), + nse.getMessage()); Review Comment: Also here. ########## core/src/main/java/org/apache/gravitino/stats/StatisticManager.java: ########## @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.stats; + +import com.google.common.collect.Lists; +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityStore; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.Relation; +import org.apache.gravitino.SupportsRelationOperations; +import org.apache.gravitino.exceptions.NoSuchEntityException; +import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; +import org.apache.gravitino.exceptions.UnmodifiableStatisticException; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.StatisticEntity; +import org.apache.gravitino.storage.IdGenerator; +import org.apache.gravitino.utils.MetadataObjectUtil; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.PrincipalUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StatisticManager { + + private static final Logger LOG = LoggerFactory.getLogger(StatisticManager.class); + + private final EntityStore store; + + private final IdGenerator idGenerator; + + public StatisticManager(EntityStore store, IdGenerator idGenerator) { + this.store = store; + this.idGenerator = idGenerator; + } + + public List<Statistic> listStatistics(String metalake, MetadataObject metadataObject) { + try { + // TODO: we will add lock when we support Iceberg, Hive statistics + NameIdentifier identifier = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType type = MetadataObjectUtil.toEntityType(metadataObject); + return store.relationOperations() + .listEntitiesByRelation( + SupportsRelationOperations.Type.METADATA_OBJECT_STAT_REL, identifier, type) + .stream() + .map( + entity -> { + StatisticEntity statisticEntity = (StatisticEntity) entity; + String name = statisticEntity.name(); + StatisticValue<?> value = statisticEntity.value(); + return new CustomStatistic(name, value); + }) + .collect(Collectors.toList()); + } catch (NoSuchEntityException nse) { + LOG.warn( + "Failed to list statistics for metadata object {}: {}", + metadataObject.fullName(), + nse.getMessage()); + throw new NoSuchMetadataObjectException( + "The metadata object of %s isn't found", metadataObject.fullName()); Review Comment: Improve the message to include the metalake name. ########## core/src/main/java/org/apache/gravitino/stats/StatisticManager.java: ########## @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.stats; + +import com.google.common.collect.Lists; +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityStore; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.Relation; +import org.apache.gravitino.SupportsRelationOperations; +import org.apache.gravitino.exceptions.NoSuchEntityException; +import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; +import org.apache.gravitino.exceptions.UnmodifiableStatisticException; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.StatisticEntity; +import org.apache.gravitino.storage.IdGenerator; +import org.apache.gravitino.utils.MetadataObjectUtil; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.PrincipalUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StatisticManager { + + private static final Logger LOG = LoggerFactory.getLogger(StatisticManager.class); + + private final EntityStore store; + + private final IdGenerator idGenerator; + + public StatisticManager(EntityStore store, IdGenerator idGenerator) { + this.store = store; + this.idGenerator = idGenerator; + } + + public List<Statistic> listStatistics(String metalake, MetadataObject metadataObject) { + try { + // TODO: we will add lock when we support Iceberg, Hive statistics + NameIdentifier identifier = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType type = MetadataObjectUtil.toEntityType(metadataObject); + return store.relationOperations() + .listEntitiesByRelation( + SupportsRelationOperations.Type.METADATA_OBJECT_STAT_REL, identifier, type) + .stream() + .map( + entity -> { + StatisticEntity statisticEntity = (StatisticEntity) entity; + String name = statisticEntity.name(); + StatisticValue<?> value = statisticEntity.value(); + return new CustomStatistic(name, value); + }) + .collect(Collectors.toList()); + } catch (NoSuchEntityException nse) { + LOG.warn( + "Failed to list statistics for metadata object {}: {}", + metadataObject.fullName(), + nse.getMessage()); + throw new NoSuchMetadataObjectException( + "The metadata object of %s isn't found", metadataObject.fullName()); + } catch (IOException ioe) { + LOG.error( + "Failed to list statistics for metadata object {}: {}", + metadataObject.fullName(), + ioe.getMessage()); + throw new RuntimeException(ioe); + } + } + + public List<Statistic> updateStatistics( + String metalake, MetadataObject metadataObject, Map<String, StatisticValue<?>> statistics) { + try { + // TODO: we will add lock when we support Iceberg, Hive statistics + NameIdentifier identifier = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType type = MetadataObjectUtil.toEntityType(metadataObject); + List<Relation> relations = Lists.newArrayList(); + List<StatisticEntity> statisticEntities = Lists.newArrayList(); + for (Map.Entry<String, StatisticValue<?>> entry : statistics.entrySet()) { + String name = entry.getKey(); + StatisticValue<?> value = entry.getValue(); + + StatisticEntity statistic = + StatisticEntity.builder() + .withId(idGenerator.nextId()) + .withName(name) + .withValue(value) + .withNamespace(Namespace.fromString(identifier.toString())) + .withAuditInfo( + AuditInfo.builder() + .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) + .withCreateTime(Instant.now()) + .withLastModifier(PrincipalUtils.getCurrentPrincipal().getName()) + .withLastModifiedTime(Instant.now()) + .build()) + .build(); + statisticEntities.add(statistic); + relations.add( + new Relation( + identifier, type, statistic.nameIdentifier(), Entity.EntityType.STATISTIC)); + } + store + .relationOperations() + .insertEntitiesAndRelations( + SupportsRelationOperations.Type.METADATA_OBJECT_STAT_REL, + statisticEntities, + relations, + true); + return statistics.entrySet().stream() + .map(entry -> new CustomStatistic(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + } catch (NoSuchEntityException nse) { + LOG.warn( + "Failed to update statistics for metadata object {}: {}", + metadataObject.fullName(), + nse.getMessage()); + throw new NoSuchMetadataObjectException( + "The metadata object of %s isn't found", metadataObject.fullName()); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + + public boolean dropStatistics( + String metalake, MetadataObject metadataObject, List<String> statistics) + throws UnmodifiableStatisticException { + try { + // TODO: we will add lock when we support Iceberg, Hive statistics + NameIdentifier identifier = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType type = MetadataObjectUtil.toEntityType(metadataObject); + List<Relation> relations = Lists.newArrayList(); + for (String statisticName : statistics) { + NameIdentifier statIdent = NameIdentifierUtil.ofStatistic(identifier, statisticName); + relations.add(new Relation(identifier, type, statIdent, Entity.EntityType.STATISTIC)); + } + + int deleteCount = + store + .relationOperations() + .deleteRelations(SupportsRelationOperations.Type.METADATA_OBJECT_STAT_REL, relations); + + // If deleteCount is 0, it means that the statistics were not found. + return deleteCount != 0; + } catch (NoSuchEntityException nse) { + LOG.warn( + "Failed to drop statistics for metadata object {}: {}", + metadataObject.fullName(), + nse.getMessage()); + throw new NoSuchMetadataObjectException( + "The metadata object of %s isn't found", metadataObject.fullName()); + } catch (IOException ioe) { + LOG.error( + "Failed to drop statistics for metadata object {}: {}", + metadataObject.fullName(), + ioe.getMessage()); + throw new RuntimeException(); + } + } + + private static class CustomStatistic implements Statistic { Review Comment: Do you need to move this to API, should this class be awared by users. ########## core/src/main/java/org/apache/gravitino/stats/StatisticManager.java: ########## @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.stats; + +import com.google.common.collect.Lists; +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityStore; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.Relation; +import org.apache.gravitino.SupportsRelationOperations; +import org.apache.gravitino.exceptions.NoSuchEntityException; +import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; +import org.apache.gravitino.exceptions.UnmodifiableStatisticException; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.StatisticEntity; +import org.apache.gravitino.storage.IdGenerator; +import org.apache.gravitino.utils.MetadataObjectUtil; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.PrincipalUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StatisticManager { + + private static final Logger LOG = LoggerFactory.getLogger(StatisticManager.class); + + private final EntityStore store; + + private final IdGenerator idGenerator; + + public StatisticManager(EntityStore store, IdGenerator idGenerator) { + this.store = store; + this.idGenerator = idGenerator; + } + + public List<Statistic> listStatistics(String metalake, MetadataObject metadataObject) { + try { + // TODO: we will add lock when we support Iceberg, Hive statistics + NameIdentifier identifier = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType type = MetadataObjectUtil.toEntityType(metadataObject); + return store.relationOperations() + .listEntitiesByRelation( + SupportsRelationOperations.Type.METADATA_OBJECT_STAT_REL, identifier, type) + .stream() + .map( + entity -> { + StatisticEntity statisticEntity = (StatisticEntity) entity; + String name = statisticEntity.name(); + StatisticValue<?> value = statisticEntity.value(); + return new CustomStatistic(name, value); + }) + .collect(Collectors.toList()); + } catch (NoSuchEntityException nse) { + LOG.warn( + "Failed to list statistics for metadata object {}: {}", + metadataObject.fullName(), + nse.getMessage()); + throw new NoSuchMetadataObjectException( + "The metadata object of %s isn't found", metadataObject.fullName()); + } catch (IOException ioe) { + LOG.error( + "Failed to list statistics for metadata object {}: {}", + metadataObject.fullName(), + ioe.getMessage()); + throw new RuntimeException(ioe); + } + } + + public List<Statistic> updateStatistics( + String metalake, MetadataObject metadataObject, Map<String, StatisticValue<?>> statistics) { + try { + // TODO: we will add lock when we support Iceberg, Hive statistics + NameIdentifier identifier = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType type = MetadataObjectUtil.toEntityType(metadataObject); + List<Relation> relations = Lists.newArrayList(); + List<StatisticEntity> statisticEntities = Lists.newArrayList(); + for (Map.Entry<String, StatisticValue<?>> entry : statistics.entrySet()) { + String name = entry.getKey(); + StatisticValue<?> value = entry.getValue(); + + StatisticEntity statistic = + StatisticEntity.builder() + .withId(idGenerator.nextId()) + .withName(name) + .withValue(value) + .withNamespace(Namespace.fromString(identifier.toString())) + .withAuditInfo( + AuditInfo.builder() + .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) + .withCreateTime(Instant.now()) + .withLastModifier(PrincipalUtils.getCurrentPrincipal().getName()) + .withLastModifiedTime(Instant.now()) + .build()) + .build(); + statisticEntities.add(statistic); + relations.add( + new Relation( + identifier, type, statistic.nameIdentifier(), Entity.EntityType.STATISTIC)); + } + store + .relationOperations() + .insertEntitiesAndRelations( + SupportsRelationOperations.Type.METADATA_OBJECT_STAT_REL, + statisticEntities, + relations, + true); + return statistics.entrySet().stream() + .map(entry -> new CustomStatistic(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + } catch (NoSuchEntityException nse) { + LOG.warn( + "Failed to update statistics for metadata object {}: {}", + metadataObject.fullName(), + nse.getMessage()); + throw new NoSuchMetadataObjectException( + "The metadata object of %s isn't found", metadataObject.fullName()); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + + public boolean dropStatistics( + String metalake, MetadataObject metadataObject, List<String> statistics) + throws UnmodifiableStatisticException { + try { + // TODO: we will add lock when we support Iceberg, Hive statistics + NameIdentifier identifier = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType type = MetadataObjectUtil.toEntityType(metadataObject); + List<Relation> relations = Lists.newArrayList(); + for (String statisticName : statistics) { + NameIdentifier statIdent = NameIdentifierUtil.ofStatistic(identifier, statisticName); + relations.add(new Relation(identifier, type, statIdent, Entity.EntityType.STATISTIC)); + } + + int deleteCount = + store + .relationOperations() + .deleteRelations(SupportsRelationOperations.Type.METADATA_OBJECT_STAT_REL, relations); Review Comment: Should you also delete the statistics, not just the relations? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
