This is an automated email from the ASF dual-hosted git repository.
lahirujayathilake pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airavata-data-catalog.git
The following commit(s) were added to refs/heads/main by this push:
new 15eee7e Implemented caching layer for data product retrievals
15eee7e is described below
commit 15eee7e3c8e4c9de0dab7d79a32840031ed88b66
Author: lahiruj <[email protected]>
AuthorDate: Fri Feb 27 11:19:41 2026 -0500
Implemented caching layer for data product retrievals
---
data-catalog-api/server/service/pom.xml | 8 ++++
.../api/DataCatalogApiServiceApplication.java | 2 +
.../datacatalog/api/mapper/DataProductMapper.java | 30 +++++++++------
.../impl/MetadataSchemaQueryExecutorImpl.java | 43 ++++++++++++++--------
.../api/service/impl/DataCatalogServiceImpl.java | 5 +++
.../src/main/resources/application.properties | 13 +++++++
.../resources/db/changelog/db.changelog-master.xml | 1 +
7 files changed, 76 insertions(+), 26 deletions(-)
diff --git a/data-catalog-api/server/service/pom.xml
b/data-catalog-api/server/service/pom.xml
index e83a61c..1e2601e 100644
--- a/data-catalog-api/server/service/pom.xml
+++ b/data-catalog-api/server/service/pom.xml
@@ -62,6 +62,14 @@
<groupId>org.liquibase</groupId>
<artifactId>liquibase-core</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-cache</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+ </dependency>
</dependencies>
<build>
diff --git
a/data-catalog-api/server/service/src/main/java/org/apache/airavata/datacatalog/api/DataCatalogApiServiceApplication.java
b/data-catalog-api/server/service/src/main/java/org/apache/airavata/datacatalog/api/DataCatalogApiServiceApplication.java
index d00a704..a630c9c 100644
---
a/data-catalog-api/server/service/src/main/java/org/apache/airavata/datacatalog/api/DataCatalogApiServiceApplication.java
+++
b/data-catalog-api/server/service/src/main/java/org/apache/airavata/datacatalog/api/DataCatalogApiServiceApplication.java
@@ -12,11 +12,13 @@ import
org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Primary;
+import org.springframework.cache.annotation.EnableCaching;
import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
@ComponentScan(basePackages = { "org.apache.airavata.datacatalog.api",
"org.apache.custos.sharing.core" })
@SpringBootApplication
+@EnableCaching
@EnableJpaRepositories({
"org.apache.custos.sharing.core.persistance.repository",
"org.apache.airavata.datacatalog.api.repository" })
@EnableJpaAuditing
diff --git
a/data-catalog-api/server/service/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java
b/data-catalog-api/server/service/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java
index 218727f..df33532 100644
---
a/data-catalog-api/server/service/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java
+++
b/data-catalog-api/server/service/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java
@@ -1,5 +1,8 @@
package org.apache.airavata.datacatalog.api.mapper;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.airavata.datacatalog.api.DataProduct;
import org.apache.airavata.datacatalog.api.exception.EntityNotFoundException;
import org.apache.airavata.datacatalog.api.model.DataProductEntity;
@@ -9,10 +12,6 @@ import
org.apache.airavata.datacatalog.api.repository.MetadataSchemaRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
/**
* Map to/from
* {@link org.apache.airavata.datacatalog.api.model.DataProductEntity}
@@ -64,18 +63,29 @@ public class DataProductMapper {
}
public void mapEntityToModel(DataProductEntity dataProductEntity,
DataProduct.Builder dataProductBuilder) {
+ mapEagerFields(dataProductEntity, dataProductBuilder);
+ if (dataProductEntity.getMetadataSchemas() != null) {
+ for (MetadataSchemaEntity metadataSchema :
dataProductEntity.getMetadataSchemas()) {
+
dataProductBuilder.addMetadataSchemas(metadataSchema.getSchemaName());
+ }
+ }
+ userInfoMapper.mapEntityToModel(dataProductEntity.getOwner(),
dataProductBuilder.getOwnerBuilder());
+ }
+
+ // Lightweight mapping for search list results. Skips metadataSchemas and
owner
+ public void mapEntityToModelForList(DataProductEntity dataProductEntity,
DataProduct.Builder dataProductBuilder) {
+ mapEagerFields(dataProductEntity, dataProductBuilder);
+ // metadataSchemas and owner intentionally skipped as they are not
needed for list views
+ }
+ // Maps fields that do not trigger lazy loads -> id, name, parentId,
metadata
+ private void mapEagerFields(DataProductEntity dataProductEntity,
DataProduct.Builder dataProductBuilder) {
dataProductBuilder
.setDataProductId(dataProductEntity.getExternalId())
.setName(dataProductEntity.getName());
if (dataProductEntity.getParentDataProductEntity() != null) {
dataProductBuilder.setParentDataProductId(dataProductEntity.getParentDataProductEntity().getExternalId());
}
- if (dataProductEntity.getMetadataSchemas() != null) {
- for (MetadataSchemaEntity metadataSchema :
dataProductEntity.getMetadataSchemas()) {
-
dataProductBuilder.addMetadataSchemas(metadataSchema.getSchemaName());
- }
- }
if (dataProductEntity.getMetadata() != null) {
ObjectMapper mapper = new ObjectMapper();
try {
@@ -84,7 +94,5 @@ public class DataProductMapper {
throw new RuntimeException(e);
}
}
-
- userInfoMapper.mapEntityToModel(dataProductEntity.getOwner(),
dataProductBuilder.getOwnerBuilder());
}
}
diff --git
a/data-catalog-api/server/service/src/main/java/org/apache/airavata/datacatalog/api/query/impl/MetadataSchemaQueryExecutorImpl.java
b/data-catalog-api/server/service/src/main/java/org/apache/airavata/datacatalog/api/query/impl/MetadataSchemaQueryExecutorImpl.java
index c1dee8d..d73f6c3 100644
---
a/data-catalog-api/server/service/src/main/java/org/apache/airavata/datacatalog/api/query/impl/MetadataSchemaQueryExecutorImpl.java
+++
b/data-catalog-api/server/service/src/main/java/org/apache/airavata/datacatalog/api/query/impl/MetadataSchemaQueryExecutorImpl.java
@@ -68,12 +68,18 @@ public class MetadataSchemaQueryExecutorImpl implements
MetadataSchemaQueryExecu
@Autowired
DataProductMapper dataProductMapper;
- @Override
- public MetadataSchemaQueryResult execute(UserEntity userEntity, String
sql, int page, int pageSize)
- throws MetadataSchemaSqlParseException,
MetadataSchemaSqlValidateException {
+ private volatile FrameworkConfig cachedConfig = null;
+ private volatile List<MetadataSchemaEntity> lastSchemas = null;
+
+ private synchronized FrameworkConfig
getOrBuildConfig(List<MetadataSchemaEntity> schemas) {
+ if (cachedConfig == null || lastSchemas != schemas) {
+ cachedConfig = buildFrameworkConfig(schemas);
+ lastSchemas = schemas;
+ }
+ return cachedConfig;
+ }
- // Create a schema that contains the data_product table and all of the
metadata
- // schemas
+ private FrameworkConfig buildFrameworkConfig(List<MetadataSchemaEntity>
metadataSchemas) {
SchemaPlus schema = Frameworks.createRootSchema(true);
schema.add("data_product", new AbstractTable() {
@Override
@@ -88,10 +94,7 @@ public class MetadataSchemaQueryExecutorImpl implements
MetadataSchemaQueryExecu
}
});
- // TODO: limit by tenant id
- List<MetadataSchemaEntity> metadataSchemas =
metadataSchemaRepository.findAll();
for (MetadataSchemaEntity metadataSchema : metadataSchemas) {
-
schema.add(metadataSchema.getSchemaName(), new AbstractTable() {
@Override
public RelDataType getRowType(RelDataTypeFactory typeFactory) {
@@ -114,14 +117,25 @@ public class MetadataSchemaQueryExecutorImpl implements
MetadataSchemaQueryExecu
});
}
- FrameworkConfig config = Frameworks.newConfigBuilder()
+ return Frameworks.newConfigBuilder()
.defaultSchema(schema)
.parserConfig(SqlParser.Config.DEFAULT.withUnquotedCasing(Casing.TO_LOWER))
.build();
+ }
+
+ @Override
+ public MetadataSchemaQueryResult execute(UserEntity userEntity, String
sql, int page, int pageSize)
+ throws MetadataSchemaSqlParseException,
MetadataSchemaSqlValidateException {
+
+ // TODO: limit by tenant id
+ List<MetadataSchemaEntity> metadataSchemas =
metadataSchemaRepository.findAll();
+
+ FrameworkConfig config = getOrBuildConfig(metadataSchemas);
Planner planner = Frameworks.getPlanner(config);
SqlNode sqlNode = parse(planner, sql);
+ SchemaPlus schema = config.getDefaultSchema();
SqlValidator validator = getValidator(schema, config, planner);
// Validate the query
@@ -172,10 +186,10 @@ public class MetadataSchemaQueryExecutorImpl implements
MetadataSchemaQueryExecu
Number countResult = (Number)
entityManager.createNativeQuery(countSql).getSingleResult();
int totalCount = countResult.intValue();
- if (page > 0 && pageSize >0) {
- // offset
- int offset = (page - 1) * pageSize;
- finalSql = finalSql + " LIMIT " + pageSize + " OFFSET " + offset;
+ if (page > 0 && pageSize > 0) {
+ // offset
+ int offset = (page - 1) * pageSize;
+ finalSql = finalSql + " LIMIT " + pageSize + " OFFSET " + offset;
}
List<DataProductEntity> dataProductEntities =
entityManager.createNativeQuery(finalSql, DataProductEntity.class)
@@ -183,9 +197,8 @@ public class MetadataSchemaQueryExecutorImpl implements
MetadataSchemaQueryExecu
List<DataProduct> dataProducts = new ArrayList<>();
for (DataProductEntity dataProductEntity : dataProductEntities) {
-
org.apache.airavata.datacatalog.api.DataProduct.Builder dpBuilder
= DataProduct.newBuilder();
- dataProductMapper.mapEntityToModel(dataProductEntity, dpBuilder);
+ dataProductMapper.mapEntityToModelForList(dataProductEntity,
dpBuilder);
dataProducts.add(dpBuilder.build());
}
return new MetadataSchemaQueryResult(dataProducts, totalCount);
diff --git
a/data-catalog-api/server/service/src/main/java/org/apache/airavata/datacatalog/api/service/impl/DataCatalogServiceImpl.java
b/data-catalog-api/server/service/src/main/java/org/apache/airavata/datacatalog/api/service/impl/DataCatalogServiceImpl.java
index d164d92..28a0a2d 100644
---
a/data-catalog-api/server/service/src/main/java/org/apache/airavata/datacatalog/api/service/impl/DataCatalogServiceImpl.java
+++
b/data-catalog-api/server/service/src/main/java/org/apache/airavata/datacatalog/api/service/impl/DataCatalogServiceImpl.java
@@ -29,6 +29,8 @@ import
org.apache.airavata.datacatalog.api.service.DataCatalogService;
import org.apache.airavata.datacatalog.api.sharing.SharingManager;
import org.apache.airavata.datacatalog.api.sharing.exception.SharingException;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.annotation.CacheEvict;
+import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -145,6 +147,7 @@ public class DataCatalogServiceImpl implements
DataCatalogService {
}
@Override
+ @Cacheable(value = "metadataSchemas", key = "'all'")
public List<MetadataSchema> getMetadataSchemas() {
return metadataSchemaRepository.findAll().stream()
.map(this::toMetadataSchema)
@@ -152,6 +155,7 @@ public class DataCatalogServiceImpl implements
DataCatalogService {
}
@Override
+ @CacheEvict(value = "metadataSchemas", allEntries = true)
public MetadataSchema createMetadataSchema(MetadataSchema metadataSchema) {
MetadataSchemaEntity metadataSchemaEntity = new MetadataSchemaEntity();
@@ -181,6 +185,7 @@ public class DataCatalogServiceImpl implements
DataCatalogService {
}
@Override
+ @CacheEvict(value = "metadataSchemas", allEntries = true)
public void deleteMetadataSchema(MetadataSchema metadataSchema) {
// TODO: check that user has write access on metadata schema
// TODO: handle metadata schema not found
diff --git
a/data-catalog-api/server/service/src/main/resources/application.properties
b/data-catalog-api/server/service/src/main/resources/application.properties
index 21051db..f3975f0 100644
--- a/data-catalog-api/server/service/src/main/resources/application.properties
+++ b/data-catalog-api/server/service/src/main/resources/application.properties
@@ -6,6 +6,19 @@ spring.jpa.hibernate.ddl-auto=validate
spring.jpa.show-sql=true
spring.liquibase.change-log=classpath:/db/changelog/db.changelog-master.xml
+# HikariCP connection pool tuning
+spring.datasource.hikari.maximum-pool-size=20
+spring.datasource.hikari.minimum-idle=5
+spring.datasource.hikari.connection-timeout=30000
+spring.datasource.hikari.idle-timeout=600000
+spring.datasource.hikari.max-lifetime=1800000
+spring.datasource.hikari.keepalive-time=30000
+spring.datasource.hikari.leak-detection-threshold=30000
+
+# Spring cache (Caffeine)
+spring.cache.type=caffeine
+spring.cache.caffeine.spec=maximumSize=100,expireAfterWrite=300s
+
# Sharing configuration
## Simple Sharing
diff --git
a/data-catalog-api/server/service/src/main/resources/db/changelog/db.changelog-master.xml
b/data-catalog-api/server/service/src/main/resources/db/changelog/db.changelog-master.xml
index 6f80e3a..8d4f585 100644
---
a/data-catalog-api/server/service/src/main/resources/db/changelog/db.changelog-master.xml
+++
b/data-catalog-api/server/service/src/main/resources/db/changelog/db.changelog-master.xml
@@ -6,4 +6,5 @@
<include file="db/changelog/2023/08/2023-08-31-init-changelog.xml"/>
<include file="db/changelog/2023/08/2023-08-31-init-custos-changelog.xml"/>
<include file="db/changelog/2023/09/2023-09-01-changelog.xml"/>
+ <include file="db/changelog/2026/02/2026-02-25-performance-indexes.xml"
relativeToChangelogFile="false"/>
</databaseChangeLog>
