This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch machristie/issue7 in repository https://gitbox.apache.org/repos/asf/airavata-data-catalog.git
commit 584cff2099e105e4fd99549ac01799e058bf4fae Author: Marcus Christie <[email protected]> AuthorDate: Wed Feb 15 15:31:19 2023 -0500 Implement adding/removing data product to/from metadata schema --- .../api/client/DataCatalogAPIClient.java | 65 ++++++++++++++++++++-- .../datacatalog/api/mapper/DataProductMapper.java | 6 ++ .../datacatalog/api/model/DataProductEntity.java | 24 ++++++++ .../api/repository/DataProductRepository.java | 1 + .../api/service/DataCatalogAPIService.java | 53 ++++++++++++++++-- .../stubs/src/main/proto/DataCatalogAPI.proto | 9 +++ 6 files changed, 146 insertions(+), 12 deletions(-) diff --git a/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java b/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java index 954f994..3d59fdb 100644 --- a/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java +++ b/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java @@ -7,11 +7,15 @@ import java.util.concurrent.TimeUnit; import org.apache.airavata.datacatalog.api.DataCatalogAPIServiceGrpc; import org.apache.airavata.datacatalog.api.DataCatalogAPIServiceGrpc.DataCatalogAPIServiceBlockingStub; import org.apache.airavata.datacatalog.api.DataProduct; +import org.apache.airavata.datacatalog.api.DataProductAddToMetadataSchemaRequest; +import org.apache.airavata.datacatalog.api.DataProductAddToMetadataSchemaResponse; import org.apache.airavata.datacatalog.api.DataProductCreateRequest; import org.apache.airavata.datacatalog.api.DataProductCreateResponse; import org.apache.airavata.datacatalog.api.DataProductDeleteRequest; import org.apache.airavata.datacatalog.api.DataProductGetRequest; import org.apache.airavata.datacatalog.api.DataProductGetResponse; +import org.apache.airavata.datacatalog.api.DataProductRemoveFromMetadataSchemaRequest; +import org.apache.airavata.datacatalog.api.DataProductRemoveFromMetadataSchemaResponse; import org.apache.airavata.datacatalog.api.DataProductUpdateRequest; import org.apache.airavata.datacatalog.api.DataProductUpdateResponse; import org.apache.airavata.datacatalog.api.FieldValueType; @@ -23,10 +27,14 @@ import org.apache.airavata.datacatalog.api.MetadataSchemaFieldCreateRequest; import org.apache.airavata.datacatalog.api.MetadataSchemaFieldCreateResponse; import org.apache.airavata.datacatalog.api.MetadataSchemaFieldListRequest; import org.apache.airavata.datacatalog.api.MetadataSchemaFieldListResponse; +import org.apache.airavata.datacatalog.api.MetadataSchemaGetRequest; +import org.apache.airavata.datacatalog.api.MetadataSchemaGetResponse; import io.grpc.Channel; import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder;; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Status; +import io.grpc.StatusRuntimeException;; public class DataCatalogAPIClient { @@ -67,6 +75,19 @@ public class DataCatalogAPIClient { return response.getMetadataSchema(); } + public MetadataSchema getMetadataSchema(String schemaName) { + MetadataSchemaGetRequest request = MetadataSchemaGetRequest.newBuilder().setSchemaName(schemaName).build(); + try { + MetadataSchemaGetResponse response = blockingStub.getMetadataSchema(request); + return response.getMetadataSchema(); + } catch (StatusRuntimeException e) { + if (e.getStatus() == Status.NOT_FOUND) { + return null; + } + throw e; + } + } + public MetadataSchemaField createMetadataSchemaField(MetadataSchemaField metadataSchemaField) { MetadataSchemaFieldCreateRequest request = MetadataSchemaFieldCreateRequest.newBuilder() .setMetadataSchemaField(metadataSchemaField).build(); @@ -81,6 +102,21 @@ public class DataCatalogAPIClient { return response.getMetadataSchemaFieldsList(); } + public DataProduct addDataProductToMetadataSchema(String dataProductId, String schemaName) { + DataProductAddToMetadataSchemaRequest request = DataProductAddToMetadataSchemaRequest.newBuilder() + .setDataProductId(dataProductId).setSchemaName(schemaName).build(); + DataProductAddToMetadataSchemaResponse response = blockingStub.addDataProductToMetadataSchema(request); + return response.getDataProduct(); + } + + public DataProduct removeDataProductFromMetadataSchema(String dataProductId, String schemaName) { + DataProductRemoveFromMetadataSchemaRequest request = DataProductRemoveFromMetadataSchemaRequest.newBuilder() + .setDataProductId(dataProductId).setSchemaName(schemaName).build(); + DataProductRemoveFromMetadataSchemaResponse response = blockingStub + .removeDataProductFromMetadataSchema(request); + return response.getDataProduct(); + } + public static void main(String[] args) throws InterruptedException { String target = "localhost:6565"; @@ -115,11 +151,19 @@ public class DataCatalogAPIClient { System.out.println( MessageFormat.format("Deleted data product with id [{0}]", result2.getDataProductId())); - MetadataSchema metadataSchema = MetadataSchema.newBuilder().setSchemaName("my_schema").build(); - metadataSchema = client.createMetadataSchema(metadataSchema); - - System.out.println( - MessageFormat.format("Created metadata schema with name [{0}]", metadataSchema.getSchemaName())); + // First check if metadata schema exists + MetadataSchema metadataSchema = client.getMetadataSchema("my_schema"); + if (metadataSchema == null) { + metadataSchema = MetadataSchema.newBuilder().setSchemaName("my_schema").build(); + metadataSchema = client.createMetadataSchema(metadataSchema); + System.out.println( + MessageFormat.format("Created metadata schema with name [{0}]", + metadataSchema.getSchemaName())); + } else { + System.out.println( + MessageFormat.format("Found metadata schema with name [{0}]", + metadataSchema.getSchemaName())); + } MetadataSchemaField field1 = MetadataSchemaField.newBuilder().setFieldName("field1") .setJsonPath("$.field1").setValueType(FieldValueType.FLOAT) @@ -142,6 +186,15 @@ public class DataCatalogAPIClient { System.out.println(MessageFormat.format("-> field {0}", field.getFieldName())); } + result = client.addDataProductToMetadataSchema(result.getDataProductId(), metadataSchema.getSchemaName()); + System.out.println(MessageFormat.format("Added data product [{0}] to metadata schema [{1}]", + result.getDataProductId(), metadataSchema.getSchemaName())); + + result = client.removeDataProductFromMetadataSchema(result.getDataProductId(), + metadataSchema.getSchemaName()); + System.out.println(MessageFormat.format("Removed data product [{0}] from metadata schema [{1}]", + result.getDataProductId(), metadataSchema.getSchemaName())); + } finally { channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS); } diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java index 8e1e155..b03fe1d 100644 --- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java +++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java @@ -2,6 +2,7 @@ package org.apache.airavata.datacatalog.api.mapper; import org.apache.airavata.datacatalog.api.DataProduct; import org.apache.airavata.datacatalog.api.model.DataProductEntity; +import org.apache.airavata.datacatalog.api.model.MetadataSchemaEntity; import org.apache.airavata.datacatalog.api.repository.DataProductRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -50,5 +51,10 @@ public class DataProductMapper { if (dataProductEntity.getParentDataProductEntity() != null) { dataProductBuilder.setParentDataProductId(dataProductEntity.getParentDataProductEntity().getExternalId()); } + if (dataProductEntity.getMetadataSchemas() != null) { + for (MetadataSchemaEntity metadataSchema : dataProductEntity.getMetadataSchemas()) { + dataProductBuilder.addSchemaName(metadataSchema.getSchemaName()); + } + } } } diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/DataProductEntity.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/DataProductEntity.java index 65b24e7..68b615f 100644 --- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/DataProductEntity.java +++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/DataProductEntity.java @@ -1,5 +1,7 @@ package org.apache.airavata.datacatalog.api.model; +import java.util.Set; + import org.hibernate.annotations.Type; import com.fasterxml.jackson.databind.JsonNode; @@ -12,6 +14,8 @@ import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; import jakarta.persistence.JoinColumn; +import jakarta.persistence.JoinTable; +import jakarta.persistence.ManyToMany; import jakarta.persistence.ManyToOne; import jakarta.persistence.SequenceGenerator; import jakarta.persistence.Table; @@ -43,6 +47,10 @@ public class DataProductEntity { @Column(name = "metadata", columnDefinition = "jsonb") private JsonNode metadata; + @ManyToMany + @JoinTable(name = "data_product_metadata_schema", joinColumns = @JoinColumn(name = "data_product_id"), inverseJoinColumns = @JoinColumn(name = "metadata_schema_id")) + private Set<MetadataSchemaEntity> metadataSchemas; + // TODO: ManyToOne mapping to owner: UserEntity public Long getDataProductId() { return dataProductId; @@ -84,6 +92,22 @@ public class DataProductEntity { this.metadata = metadata; } + public Set<MetadataSchemaEntity> getMetadataSchemas() { + return metadataSchemas; + } + + public void setMetadataSchemas(Set<MetadataSchemaEntity> metadataSchemas) { + this.metadataSchemas = metadataSchemas; + } + + public void addMetadataSchema(MetadataSchemaEntity metadataSchema) { + this.metadataSchemas.add(metadataSchema); + } + + public void removeMetadataSchema(MetadataSchemaEntity metadataSchema) { + this.metadataSchemas.remove(metadataSchema); + } + @Override public int hashCode() { final int prime = 31; diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/DataProductRepository.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/DataProductRepository.java index acec036..426e4d4 100644 --- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/DataProductRepository.java +++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/DataProductRepository.java @@ -4,6 +4,7 @@ import org.apache.airavata.datacatalog.api.model.DataProductEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.transaction.annotation.Transactional; +@Transactional(readOnly = true) public interface DataProductRepository extends JpaRepository<DataProductEntity, Long> { DataProductEntity findByExternalId(String externalId); diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java index 3869ee5..7374def 100644 --- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java +++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java @@ -28,6 +28,8 @@ import org.apache.airavata.datacatalog.api.MetadataSchemaFieldListRequest; import org.apache.airavata.datacatalog.api.MetadataSchemaFieldListResponse; import org.apache.airavata.datacatalog.api.MetadataSchemaFieldUpdateRequest; import org.apache.airavata.datacatalog.api.MetadataSchemaFieldUpdateResponse; +import org.apache.airavata.datacatalog.api.MetadataSchemaGetRequest; +import org.apache.airavata.datacatalog.api.MetadataSchemaGetResponse; import org.apache.airavata.datacatalog.api.mapper.DataProductMapper; import org.apache.airavata.datacatalog.api.mapper.MetadataSchemaFieldMapper; import org.apache.airavata.datacatalog.api.mapper.MetadataSchemaMapper; @@ -41,10 +43,13 @@ import org.lognet.springboot.grpc.GRpcService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; +import io.grpc.Status; import io.grpc.stub.StreamObserver; @GRpcService +@Transactional public class DataCatalogAPIService extends DataCatalogAPIServiceGrpc.DataCatalogAPIServiceImplBase { private static final Logger logger = LoggerFactory.getLogger(DataCatalogAPIService.class); @@ -76,7 +81,7 @@ public class DataCatalogAPIService extends DataCatalogAPIServiceGrpc.DataCatalog DataProductEntity dataProductEntity = new DataProductEntity(); dataProductEntity.setExternalId(UUID.randomUUID().toString()); dataProductMapper.mapModelToEntity(request.getDataProduct(), dataProductEntity); - DataProductEntity savedDataProductEntity = dataProductRepository.save(dataProductEntity); + DataProductEntity savedDataProductEntity = dataProductRepository.saveAndFlush(dataProductEntity); // TODO: SharingManager.grantPermissionToUser(userInfo, dataProduct, // Permission.OWNER) @@ -131,8 +136,33 @@ public class DataCatalogAPIService extends DataCatalogAPIServiceGrpc.DataCatalog @Override public void addDataProductToMetadataSchema(DataProductAddToMetadataSchemaRequest request, StreamObserver<DataProductAddToMetadataSchemaResponse> responseObserver) { - // TODO Auto-generated method stub - super.addDataProductToMetadataSchema(request, responseObserver); + String dataProductId = request.getDataProductId(); + // TODO: handle data product not found + DataProductEntity dataProduct = dataProductRepository.findByExternalId(dataProductId); + String schemaName = request.getSchemaName(); + // TODO: handle metadata schema not found + MetadataSchemaEntity metadataSchemaEntity = metadataSchemaRepository.findBySchemaName(schemaName); + dataProduct.addMetadataSchema(metadataSchemaEntity); + + DataProductAddToMetadataSchemaResponse.Builder responseBuilder = DataProductAddToMetadataSchemaResponse + .newBuilder(); + dataProductMapper.mapEntityToModel(dataProduct, responseBuilder.getDataProductBuilder()); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void getMetadataSchema(MetadataSchemaGetRequest request, + StreamObserver<MetadataSchemaGetResponse> responseObserver) { + MetadataSchemaEntity metadataSchemaEntity = metadataSchemaRepository.findBySchemaName(request.getSchemaName()); + if (metadataSchemaEntity == null) { + responseObserver.onError(Status.NOT_FOUND.asException()); + responseObserver.onCompleted(); + } + MetadataSchemaGetResponse.Builder responseBuilder = MetadataSchemaGetResponse.newBuilder(); + metadataSchemaMapper.mapEntityToModel(metadataSchemaEntity, responseBuilder.getMetadataSchemaBuilder()); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); } @Override @@ -193,15 +223,26 @@ public class DataCatalogAPIService extends DataCatalogAPIServiceGrpc.DataCatalog metadataSchemaFieldMapper.mapEntityToModel(metadataSchemaFieldEntity, builder); } responseObserver.onNext(responseBuilder.build()); - responseObserver.onError(null); responseObserver.onCompleted(); } @Override public void removeDataProductFromMetadataSchema(DataProductRemoveFromMetadataSchemaRequest request, StreamObserver<DataProductRemoveFromMetadataSchemaResponse> responseObserver) { - // TODO Auto-generated method stub - super.removeDataProductFromMetadataSchema(request, responseObserver); + + String dataProductId = request.getDataProductId(); + // TODO: handle data product not found + DataProductEntity dataProduct = dataProductRepository.findByExternalId(dataProductId); + String schemaName = request.getSchemaName(); + // TODO: handle metadata schema not found + MetadataSchemaEntity metadataSchemaEntity = metadataSchemaRepository.findBySchemaName(schemaName); + dataProduct.removeMetadataSchema(metadataSchemaEntity); + + DataProductRemoveFromMetadataSchemaResponse.Builder responseBuilder = DataProductRemoveFromMetadataSchemaResponse + .newBuilder(); + dataProductMapper.mapEntityToModel(dataProduct, responseBuilder.getDataProductBuilder()); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); } @Override diff --git a/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto b/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto index b1d40dd..cb03016 100644 --- a/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto +++ b/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto @@ -62,6 +62,7 @@ message DataProduct { optional string parent_data_product_id = 2; string name = 3; optional string metadata = 4; + repeated string schema_name = 5; } enum FieldValueType { @@ -105,6 +106,13 @@ message DataProductDeleteRequest { message DataProductDeleteResponse { } +message MetadataSchemaGetRequest { + UserInfo user_info = 1; + string schema_name = 2; +} +message MetadataSchemaGetResponse { + MetadataSchema metadata_schema = 1; +} message MetadataSchemaCreateRequest { UserInfo user_info = 1; MetadataSchema metadata_schema = 2; @@ -168,6 +176,7 @@ service DataCatalogAPIService { rpc updateDataProduct(DataProductUpdateRequest) returns (DataProductUpdateResponse){} rpc getDataProduct(DataProductGetRequest) returns (DataProductGetResponse){} rpc deleteDataProduct(DataProductDeleteRequest) returns (DataProductDeleteResponse){} + rpc getMetadataSchema(MetadataSchemaGetRequest) returns (MetadataSchemaGetResponse){} rpc createMetadataSchema(MetadataSchemaCreateRequest) returns (MetadataSchemaCreateResponse){} rpc deleteMetadataSchema(MetadataSchemaCreateRequest) returns (MetadataSchemaCreateResponse){} rpc createMetadataSchemaField(MetadataSchemaFieldCreateRequest) returns (MetadataSchemaFieldCreateResponse){}
