This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6952-f8e58461e6ed805e0bd15b52da946e42692a5ccd in repository https://gitbox.apache.org/repos/asf/texera.git
commit 74abf0e14a5436066092bd30345e2cc37b3717eb Author: Xuan Gu <[email protected]> AuthorDate: Sun Aug 2 20:16:05 2026 -0700 feat(dataset): add contributor metadata storage and API (#6952) ### What changes were proposed in this PR? This PR adds contributor metadata to datasets so that dataset authors and other contributors can be properly acknowledged. Each dataset can record a list of contributors, where each contributor has a name, a creator flag, and optional email, affiliation, and free-text comments fields. Changes: - Adds a new `dataset_contributor` table with a foreign key to `dataset` and cascade deletion in `texera_ddl.sql`, along with the incremental migration script `sql/updates/30.sql` and changelog entry 30. - Updates `file-service` so that contributors are: - returned with the dataset detail response, - accepted during dataset creation, and - replaceable through a new `POST /dataset/update/contributors` endpoint (with `did` in the request body, matching the existing `update/description` and `update/name` endpoints), protected by the existing WRITE-access check. - Validates that each contributor has a name, and that name, email, and affiliation do not exceed 256 characters. - Models email, affiliation, and comments as optional in the API: omitted fields deserialize to null. Currently, contributors are stored independently for each dataset. Cross-dataset contributor identity is left as a future extension. The UI will be added in a follow-up PR. ### Any related issues, documentation, discussions? Related to #6926 ### How was this PR tested? 7 ScalaTest cases were added, covering persisting contributors at dataset creation, replacing and clearing the list, validation failures (missing name, over-length fields), the WRITE-access check, and cascade deletion when the dataset is removed. ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Claude Code (Claude Fable 5) Co-authored-by: Claude Fable 5 <[email protected]> --- .../texera/service/resource/DatasetResource.scala | 105 +++++++++- .../service/resource/DatasetResourceSpec.scala | 225 +++++++++++++++++++++ sql/changelog.xml | 5 + sql/texera_ddl.sql | 14 ++ sql/updates/30.sql | 38 ++++ 5 files changed, 383 insertions(+), 4 deletions(-) diff --git a/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala b/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala index c376e4ce04..ce872f4520 100644 --- a/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala +++ b/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala @@ -34,6 +34,7 @@ import org.apache.texera.dao.SqlServer import org.apache.texera.dao.SqlServer.withTransaction import org.apache.texera.dao.jooq.generated.enums.PrivilegeEnum import org.apache.texera.dao.jooq.generated.tables.Dataset.DATASET +import org.apache.texera.dao.jooq.generated.tables.DatasetContributor.DATASET_CONTRIBUTOR import org.apache.texera.dao.jooq.generated.tables.DatasetUserAccess.DATASET_USER_ACCESS import org.apache.texera.dao.jooq.generated.tables.DatasetVersion.DATASET_VERSION import org.apache.texera.dao.jooq.generated.tables.User.USER @@ -175,12 +176,74 @@ object DatasetResource { normalized } + /** + * Helper function to get the contributors using the did + */ + def getContributorsByDid(ctx: DSLContext, did: Integer): List[Contributor] = { + ctx + .selectFrom(DATASET_CONTRIBUTOR) + .where(DATASET_CONTRIBUTOR.DID.eq(did)) + .fetch() + .asScala + .toList + .map { record => + Contributor( + name = record.getName, + creator = record.getCreator, + affiliation = record.getAffiliation, + email = record.getEmail, + comments = record.getComments + ) + } + } + + /** + * Helper function to insert the contributors of a dataset in one batch + */ + def insertContributors(ctx: DSLContext, did: Integer, contributors: List[Contributor]): Unit = { + val records = contributors.map { contributor => + if (contributor == null || contributor.name == null || contributor.name.trim.isEmpty) { + throw new BadRequestException("Each contributor must have a name") + } + if ( + contributor.name.length > 256 || + Option(contributor.email).exists(_.length > 256) || + Option(contributor.affiliation).exists(_.length > 256) + ) { + throw new BadRequestException("Contributor fields must not exceed 256 characters") + } + val record = ctx.newRecord(DATASET_CONTRIBUTOR) + record.setDid(did) + record.setName(contributor.name) + record.setCreator(contributor.creator) + record.setAffiliation(contributor.affiliation) + record.setEmail(contributor.email) + record.setComments(contributor.comments) + record + } + ctx.batchInsert(records.asJava).execute() + } + + case class Contributor( + name: String, + creator: Boolean = false, + affiliation: String = null, + email: String = null, + comments: String = null + ) + + case class DatasetContributorsModification( + did: Integer, + contributors: Option[List[Contributor]] = None + ) + case class DashboardDataset( dataset: Dataset, ownerEmail: String, accessPrivilege: EnumType, isOwner: Boolean, - size: Long + size: Long, + contributors: List[Contributor] = Nil ) case class DashboardDatasetVersion( @@ -192,7 +255,8 @@ object DatasetResource { datasetName: String, datasetDescription: String, isDatasetPublic: Boolean, - isDatasetDownloadable: Boolean + isDatasetDownloadable: Boolean, + contributors: Option[List[Contributor]] = None ) case class Diff( @@ -257,7 +321,8 @@ class DatasetResource extends LazyLogging { isOwner, withLakeFSErrorHandling(s"retrieving the size of dataset '${targetDataset.getName}'") { LakeFSStorageClient.retrieveRepositorySize(targetDataset.getRepositoryName) - } + }, + contributors = DatasetResource.getContributorsByDid(ctx, did) ) } @@ -309,6 +374,9 @@ class DatasetResource extends LazyLogging { .fetchOne() } + val savedContributors = request.contributors.getOrElse(Nil) + DatasetResource.insertContributors(ctx, createdDataset.getDid, savedContributors) + // Initialize the repository in LakeFS val repositoryName = s"dataset-${createdDataset.getDid}" try { @@ -347,7 +415,8 @@ class DatasetResource extends LazyLogging { user.getEmail, PrivilegeEnum.WRITE, isOwner = true, - 0 + 0, + savedContributors ) } } @@ -494,6 +563,34 @@ class DatasetResource extends LazyLogging { } } + @POST + @Consumes(Array(MediaType.APPLICATION_JSON)) + @Produces(Array(MediaType.APPLICATION_JSON)) + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/update/contributors") + def updateDatasetContributors( + modificator: DatasetContributorsModification, + @Auth user: SessionUser + ): Response = { + withTransaction(context) { ctx => + if (!userHasWriteAccess(ctx, modificator.did, user.getUid)) { + throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE) + } + + ctx + .delete(DATASET_CONTRIBUTOR) + .where(DATASET_CONTRIBUTOR.DID.eq(modificator.did)) + .execute() + DatasetResource.insertContributors( + ctx, + modificator.did, + modificator.contributors.getOrElse(Nil) + ) + + Response.ok().build() + } + } + @POST @Consumes(Array(MediaType.APPLICATION_JSON)) @Produces(Array(MediaType.APPLICATION_JSON)) diff --git a/file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourceSpec.scala b/file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourceSpec.scala index fb27fa6731..7ca1e9429c 100644 --- a/file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourceSpec.scala +++ b/file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourceSpec.scala @@ -343,6 +343,40 @@ class DatasetResourceSpec dashboardDataset.dataset.getIsDownloadable shouldBe false } + it should "persist contributors and return them in the response" in { + val contributors = List( + DatasetResource.Contributor( + name = "test1", + creator = true, + affiliation = "Test Lab A", + email = "[email protected]", + comments = "collected the data" + ), + DatasetResource.Contributor( + name = "test2", + creator = false, + affiliation = "Test Lab B", + email = "[email protected]", + comments = null + ) + ) + val createDatasetRequest = DatasetResource.CreateDatasetRequest( + datasetName = "contributor-ds", + datasetDescription = "dataset with contributors", + isDatasetPublic = false, + isDatasetDownloadable = true, + contributors = Some(contributors) + ) + + val createdDataset = datasetResource.createDataset(createDatasetRequest, sessionUser) + + createdDataset.contributors should contain theSameElementsAs contributors + DatasetResource.getContributorsByDid( + getDSLContext, + createdDataset.dataset.getDid + ) should contain theSameElementsAs contributors + } + it should "delete dataset successfully if user owns it" in { val dataset = new Dataset dataset.setName("delete-ds") @@ -404,6 +438,197 @@ class DatasetResourceSpec dashboardDataset.size should be >= 0L } + "updateDatasetContributors" should "replace the contributor list of the dataset" in { + val initial = List( + DatasetResource.Contributor( + "test1", + creator = true, + "Test Lab A", + "[email protected]", + "initial" + ) + ) + val createdDataset = datasetResource.createDataset( + DatasetResource.CreateDatasetRequest( + datasetName = "contributor-update-ds", + datasetDescription = "dataset for contributor update", + isDatasetPublic = false, + isDatasetDownloadable = true, + contributors = Some(initial) + ), + sessionUser + ) + val did = createdDataset.dataset.getDid + + val replacement = List( + DatasetResource + .Contributor("test2", creator = false, "Test Lab C", "[email protected]", "curation"), + DatasetResource.Contributor( + "test3", + creator = true, + "Test Lab D", + "[email protected]", + "analysis" + ) + ) + val response = datasetResource.updateDatasetContributors( + DatasetResource.DatasetContributorsModification(did, Some(replacement)), + sessionUser + ) + + response.getStatus shouldEqual 200 + DatasetResource.getContributorsByDid( + getDSLContext, + did + ) should contain theSameElementsAs replacement + } + + it should "clear all contributors when given an empty list" in { + val createdDataset = datasetResource.createDataset( + DatasetResource.CreateDatasetRequest( + datasetName = "contributor-clear-ds", + datasetDescription = "dataset for contributor clear test", + isDatasetPublic = false, + isDatasetDownloadable = true, + contributors = Some( + List( + DatasetResource + .Contributor("test1", creator = true, "Test Lab A", "[email protected]", null), + DatasetResource + .Contributor("test2", creator = false, "Test Lab B", "[email protected]", null) + ) + ) + ), + sessionUser + ) + val did = createdDataset.dataset.getDid + DatasetResource.getContributorsByDid(getDSLContext, did) should have size 2 + + val response = datasetResource.updateDatasetContributors( + DatasetResource.DatasetContributorsModification(did, Some(Nil)), + sessionUser + ) + + response.getStatus shouldEqual 200 + DatasetResource.getContributorsByDid(getDSLContext, did) shouldBe empty + } + + it should "reject a contributor without a name" in { + val createdDataset = datasetResource.createDataset( + DatasetResource.CreateDatasetRequest( + datasetName = "contributor-noname-ds", + datasetDescription = "dataset for contributor validation test", + isDatasetPublic = false, + isDatasetDownloadable = true + ), + sessionUser + ) + val did = createdDataset.dataset.getDid + + assertThrows[BadRequestException] { + datasetResource.updateDatasetContributors( + DatasetResource.DatasetContributorsModification( + did, + Some( + List( + DatasetResource + .Contributor(null, creator = false, "Test Lab A", "[email protected]", null) + ) + ) + ), + sessionUser + ) + } + DatasetResource.getContributorsByDid(getDSLContext, did) shouldBe empty + } + + it should "reject contributor fields longer than 256 characters" in { + val createdDataset = datasetResource.createDataset( + DatasetResource.CreateDatasetRequest( + datasetName = "contributor-toolong-ds", + datasetDescription = "dataset for contributor length test", + isDatasetPublic = false, + isDatasetDownloadable = true + ), + sessionUser + ) + + assertThrows[BadRequestException] { + datasetResource.updateDatasetContributors( + DatasetResource.DatasetContributorsModification( + createdDataset.dataset.getDid, + Some( + List( + DatasetResource + .Contributor( + "A" * 257, + creator = false, + "Test Lab A", + "[email protected]", + null + ) + ) + ) + ), + sessionUser + ) + } + } + + it should "refuse to update contributors without write access" in { + val createdDataset = datasetResource.createDataset( + DatasetResource.CreateDatasetRequest( + datasetName = "contributor-forbidden-ds", + datasetDescription = "dataset for contributor permission test", + isDatasetPublic = true, + isDatasetDownloadable = true + ), + sessionUser + ) + val did = createdDataset.dataset.getDid + + assertThrows[ForbiddenException] { + datasetResource.updateDatasetContributors( + DatasetResource.DatasetContributorsModification( + did, + Some( + List( + DatasetResource + .Contributor("test1", creator = false, "Test Lab E", "[email protected]", null) + ) + ) + ), + multipartNoWriteSessionUser + ) + } + + DatasetResource.getContributorsByDid(getDSLContext, did) shouldBe empty + } + + it should "remove contributors when their dataset is deleted" in { + val createdDataset = datasetResource.createDataset( + DatasetResource.CreateDatasetRequest( + datasetName = "contributor-cascade-ds", + datasetDescription = "dataset for contributor cascade test", + isDatasetPublic = false, + isDatasetDownloadable = true, + contributors = Some( + List( + DatasetResource + .Contributor("test1", creator = true, "Test Lab A", "[email protected]", null) + ) + ) + ), + sessionUser + ) + val did = createdDataset.dataset.getDid + DatasetResource.getContributorsByDid(getDSLContext, did) should have size 1 + + datasetResource.deleteDataset(did, sessionUser).getStatus shouldEqual 200 + + DatasetResource.getContributorsByDid(getDSLContext, did) shouldBe empty + } + "findExistingUploadFiles" should "match committed and staged files by path and size" in { val repoName = s"existing-upload-${System.nanoTime()}" val dataset = new Dataset diff --git a/sql/changelog.xml b/sql/changelog.xml index 2cec53da2f..0288dbd6b8 100644 --- a/sql/changelog.xml +++ b/sql/changelog.xml @@ -58,6 +58,11 @@ <sqlFile path="sql/updates/29.sql"/> </changeSet> + <!-- Add dataset_contributor table (contributor metadata for datasets) --> + <changeSet id="30" author="xuang7"> + <sqlFile path="sql/updates/30.sql"/> + </changeSet> + <!-- example changeSet <changeSet id="1" author="author"> <sqlFile path="sql/updates/1.sql"/> diff --git a/sql/texera_ddl.sql b/sql/texera_ddl.sql index 4132aba32c..982f784a47 100644 --- a/sql/texera_ddl.sql +++ b/sql/texera_ddl.sql @@ -65,6 +65,7 @@ DROP TABLE IF EXISTS dataset_upload_session_part CASCADE; DROP TABLE IF EXISTS dataset CASCADE; DROP TABLE IF EXISTS dataset_user_access CASCADE; DROP TABLE IF EXISTS dataset_version CASCADE; +DROP TABLE IF EXISTS dataset_contributor CASCADE; DROP TABLE IF EXISTS public_project CASCADE; DROP TABLE IF EXISTS project_user_access CASCADE; DROP TABLE IF EXISTS workflow_user_likes CASCADE; @@ -314,6 +315,19 @@ CREATE TABLE IF NOT EXISTS dataset_version FOREIGN KEY (did) REFERENCES dataset(did) ON DELETE CASCADE ); +-- dataset_contributor +CREATE TABLE IF NOT EXISTS dataset_contributor +( + cid SERIAL PRIMARY KEY, + did INT NOT NULL, + name VARCHAR(256) NOT NULL, + creator BOOLEAN NOT NULL DEFAULT FALSE, + email VARCHAR(256), + affiliation VARCHAR(256), + comments TEXT, + FOREIGN KEY (did) REFERENCES dataset(did) ON DELETE CASCADE + ); + CREATE TABLE IF NOT EXISTS dataset_upload_session ( did INT NOT NULL, diff --git a/sql/updates/30.sql b/sql/updates/30.sql new file mode 100644 index 0000000000..e3d06ce53a --- /dev/null +++ b/sql/updates/30.sql @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +\c texera_db + +SET search_path TO texera_db; + +BEGIN; + +CREATE TABLE IF NOT EXISTS dataset_contributor +( + cid SERIAL PRIMARY KEY, + did INT NOT NULL, + name VARCHAR(256) NOT NULL, + creator BOOLEAN NOT NULL DEFAULT FALSE, + email VARCHAR(256), + affiliation VARCHAR(256), + comments TEXT, + FOREIGN KEY (did) REFERENCES dataset(did) ON DELETE CASCADE +); + +COMMIT;
