This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/branch-0.9 by this push:
new 07914561b3 [#6893] feat(server): support fileset multiple locations in
REST API (#6933)
07914561b3 is described below
commit 07914561b3428e17ebfa6f0bb9d85d1a73522f68
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Apr 15 15:21:35 2025 +0800
[#6893] feat(server): support fileset multiple locations in REST API (#6933)
### What changes were proposed in this pull request?
support fileset multiple locations in REST API
### Why are the changes needed?
support fileset multiple locations in REST API
Fix: #6893
### Does this PR introduce _any_ user-facing change?
yes
### How was this patch tested?
tests added
Co-authored-by: mchades <[email protected]>
---
.../gravitino/client/TestFilesetCatalog.java | 5 +-
.../gravitino/client/TestSupportCredentials.java | 4 +-
.../apache/gravitino/client/TestSupportRoles.java | 4 +-
.../apache/gravitino/client/TestSupportTags.java | 4 +-
.../filesystem/hadoop/GravitinoMockServerBase.java | 7 +-
.../gravitino/filesystem/hadoop/TestGvfsBase.java | 3 +-
.../org/apache/gravitino/dto/file/FilesetDTO.java | 156 ++++++++++++++++++---
.../dto/requests/FilesetCreateRequest.java | 4 +
.../apache/gravitino/dto/util/DTOConverters.java | 2 +-
.../apache/gravitino/dto/file/TestFilesetDTO.java | 98 +++++++++++++
docs/open-api/filesets.yaml | 15 ++
.../server/web/rest/FilesetOperations.java | 31 +++-
.../server/web/rest/TestFilesetOperations.java | 135 ++++++++++++++++--
13 files changed, 423 insertions(+), 45 deletions(-)
diff --git
a/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java
b/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java
index f45adfab5f..6597ed5c59 100644
---
a/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java
+++
b/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java
@@ -18,6 +18,7 @@
*/
package org.apache.gravitino.client;
+import static org.apache.gravitino.file.Fileset.LOCATION_NAME_UNKNOWN;
import static org.apache.hc.core5.http.HttpStatus.SC_CONFLICT;
import static org.apache.hc.core5.http.HttpStatus.SC_NOT_FOUND;
import static org.apache.hc.core5.http.HttpStatus.SC_OK;
@@ -544,11 +545,13 @@ public class TestFilesetCatalog extends TestBase {
String comment,
String location,
Map<String, String> properties) {
+ Map<String, String> locations =
+ location == null ? ImmutableMap.of() :
ImmutableMap.of(LOCATION_NAME_UNKNOWN, location);
return FilesetDTO.builder()
.name(name)
.type(type)
.comment(comment)
- .storageLocation(location)
+ .storageLocations(locations)
.properties(properties)
.audit(AuditDTO.builder().withCreator("creator").withCreateTime(Instant.now()).build())
.build();
diff --git
a/clients/client-java/src/test/java/org/apache/gravitino/client/TestSupportCredentials.java
b/clients/client-java/src/test/java/org/apache/gravitino/client/TestSupportCredentials.java
index 842af4a640..b2f305fbf0 100644
---
a/clients/client-java/src/test/java/org/apache/gravitino/client/TestSupportCredentials.java
+++
b/clients/client-java/src/test/java/org/apache/gravitino/client/TestSupportCredentials.java
@@ -18,10 +18,12 @@
*/
package org.apache.gravitino.client;
+import static org.apache.gravitino.file.Fileset.LOCATION_NAME_UNKNOWN;
import static org.apache.hc.core5.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
import static org.apache.hc.core5.http.HttpStatus.SC_OK;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.collect.ImmutableMap;
import java.util.Collections;
import java.util.Locale;
import org.apache.gravitino.Catalog;
@@ -74,7 +76,7 @@ public class TestSupportCredentials extends TestBase {
.name("fileset1")
.comment("comment1")
.type(Fileset.Type.EXTERNAL)
- .storageLocation("s3://bucket/path")
+ .storageLocations(ImmutableMap.of(LOCATION_NAME_UNKNOWN,
"s3://bucket/path"))
.properties(Collections.emptyMap())
.audit(AuditDTO.builder().withCreator("test").build())
.build(),
diff --git
a/clients/client-java/src/test/java/org/apache/gravitino/client/TestSupportRoles.java
b/clients/client-java/src/test/java/org/apache/gravitino/client/TestSupportRoles.java
index b22d5b21b4..ceb6bec575 100644
---
a/clients/client-java/src/test/java/org/apache/gravitino/client/TestSupportRoles.java
+++
b/clients/client-java/src/test/java/org/apache/gravitino/client/TestSupportRoles.java
@@ -18,11 +18,13 @@
*/
package org.apache.gravitino.client;
+import static org.apache.gravitino.file.Fileset.LOCATION_NAME_UNKNOWN;
import static org.apache.hc.core5.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
import static org.apache.hc.core5.http.HttpStatus.SC_NOT_FOUND;
import static org.apache.hc.core5.http.HttpStatus.SC_OK;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.collect.ImmutableMap;
import java.util.Collections;
import java.util.Locale;
import org.apache.gravitino.Catalog;
@@ -142,7 +144,7 @@ public class TestSupportRoles extends TestBase {
.name("fileset1")
.comment("comment1")
.type(Fileset.Type.EXTERNAL)
- .storageLocation("s3://bucket/path")
+ .storageLocations(ImmutableMap.of(LOCATION_NAME_UNKNOWN,
"s3://bucket/path"))
.properties(Collections.emptyMap())
.audit(AuditDTO.builder().withCreator("test").build())
.build(),
diff --git
a/clients/client-java/src/test/java/org/apache/gravitino/client/TestSupportTags.java
b/clients/client-java/src/test/java/org/apache/gravitino/client/TestSupportTags.java
index 3d903a972c..157f69191e 100644
---
a/clients/client-java/src/test/java/org/apache/gravitino/client/TestSupportTags.java
+++
b/clients/client-java/src/test/java/org/apache/gravitino/client/TestSupportTags.java
@@ -18,11 +18,13 @@
*/
package org.apache.gravitino.client;
+import static org.apache.gravitino.file.Fileset.LOCATION_NAME_UNKNOWN;
import static org.apache.hc.core5.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
import static org.apache.hc.core5.http.HttpStatus.SC_NOT_FOUND;
import static org.apache.hc.core5.http.HttpStatus.SC_OK;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.collect.ImmutableMap;
import java.util.Collections;
import java.util.Locale;
import org.apache.gravitino.Catalog;
@@ -152,7 +154,7 @@ public class TestSupportTags extends TestBase {
.name("fileset1")
.comment("comment1")
.type(Fileset.Type.EXTERNAL)
- .storageLocation("s3://bucket/path")
+ .storageLocations(ImmutableMap.of(LOCATION_NAME_UNKNOWN,
"s3://bucket/path"))
.properties(Collections.emptyMap())
.audit(AuditDTO.builder().withCreator("test").build())
.build(),
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/GravitinoMockServerBase.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/GravitinoMockServerBase.java
index da2be0caef..531cb37eb1 100644
---
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/GravitinoMockServerBase.java
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/GravitinoMockServerBase.java
@@ -18,6 +18,7 @@
*/
package org.apache.gravitino.filesystem.hadoop;
+import static org.apache.gravitino.file.Fileset.LOCATION_NAME_UNKNOWN;
import static org.apache.hc.core5.http.HttpStatus.SC_OK;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -184,11 +185,15 @@ public abstract class GravitinoMockServerBase {
String.format(
"/api/metalakes/%s/catalogs/%s/schemas/%s/filesets/%s",
metalakeName, catalogName, schemaName, filesetName);
+ Map<String, String> locations =
+ location == null
+ ? Collections.emptyMap()
+ : ImmutableMap.of(LOCATION_NAME_UNKNOWN, location);
FilesetDTO mockFileset =
FilesetDTO.builder()
.name(fileset.name())
.type(type)
- .storageLocation(location)
+ .storageLocations(locations)
.comment("comment")
.properties(ImmutableMap.of("k1", "v1"))
.audit(AuditDTO.builder().withCreator("creator").withCreateTime(Instant.now()).build())
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestGvfsBase.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestGvfsBase.java
index be30e42a4b..9c8a863eaa 100644
---
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestGvfsBase.java
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/TestGvfsBase.java
@@ -18,6 +18,7 @@
*/
package org.apache.gravitino.filesystem.hadoop;
+import static org.apache.gravitino.file.Fileset.LOCATION_NAME_UNKNOWN;
import static org.apache.hc.core5.http.HttpStatus.SC_OK;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -341,7 +342,7 @@ public class TestGvfsBase extends GravitinoMockServerBase {
.comment("comment")
.type(Fileset.Type.MANAGED)
.audit(AuditDTO.builder().build())
- .storageLocation(filesetLocation.toString())
+ .storageLocations(ImmutableMap.of(LOCATION_NAME_UNKNOWN,
filesetLocation))
.build());
CredentialResponse credentialResponse = new CredentialResponse(new
CredentialDTO[] {});
diff --git a/common/src/main/java/org/apache/gravitino/dto/file/FilesetDTO.java
b/common/src/main/java/org/apache/gravitino/dto/file/FilesetDTO.java
index 7630b4c8d0..eb6a4ffef2 100644
--- a/common/src/main/java/org/apache/gravitino/dto/file/FilesetDTO.java
+++ b/common/src/main/java/org/apache/gravitino/dto/file/FilesetDTO.java
@@ -20,20 +20,16 @@ package org.apache.gravitino.dto.file;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import java.util.HashMap;
import java.util.Map;
import javax.annotation.Nullable;
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.apache.gravitino.dto.AuditDTO;
import org.apache.gravitino.file.Fileset;
/** Represents a Fileset DTO (Data Transfer Object). */
-@NoArgsConstructor(access = AccessLevel.PRIVATE, force = true)
-@AllArgsConstructor(access = AccessLevel.PRIVATE)
@EqualsAndHashCode
public class FilesetDTO implements Fileset {
@@ -49,12 +45,34 @@ public class FilesetDTO implements Fileset {
@JsonProperty("storageLocation")
private String storageLocation;
+ @JsonProperty("storageLocations")
+ private Map<String, String> storageLocations;
+
@JsonProperty("properties")
private Map<String, String> properties;
@JsonProperty("audit")
private AuditDTO audit;
+ private FilesetDTO() {}
+
+ private FilesetDTO(
+ String name,
+ String comment,
+ Type type,
+ String storageLocation,
+ Map<String, String> storageLocations,
+ Map<String, String> properties,
+ AuditDTO audit) {
+ this.name = name;
+ this.comment = comment;
+ this.type = type;
+ this.storageLocation = storageLocation;
+ this.storageLocations = storageLocations;
+ this.properties = properties;
+ this.audit = audit;
+ }
+
@Override
public String name() {
return name;
@@ -72,8 +90,8 @@ public class FilesetDTO implements Fileset {
}
@Override
- public String storageLocation() {
- return storageLocation;
+ public Map<String, String> storageLocations() {
+ return storageLocations;
}
@Override
@@ -86,18 +104,116 @@ public class FilesetDTO implements Fileset {
return audit;
}
- @Builder(builderMethodName = "builder")
- private static FilesetDTO internalBuilder(
- String name,
- String comment,
- Type type,
- String storageLocation,
- Map<String, String> properties,
- AuditDTO audit) {
- Preconditions.checkArgument(StringUtils.isNotBlank(name), "name cannot be
null or empty");
- Preconditions.checkNotNull(type, "type cannot be null");
- Preconditions.checkNotNull(audit, "audit cannot be null");
+ /**
+ * Create a new FilesetDTO builder.
+ *
+ * @return A new FilesetDTO builder.
+ */
+ public static FilesetDTOBuilder builder() {
+ return new FilesetDTOBuilder();
+ }
- return new FilesetDTO(name, comment, type, storageLocation, properties,
audit);
+ /** Builder for FilesetDTO. */
+ public static class FilesetDTOBuilder {
+ private String name;
+ private String comment;
+ private Type type;
+ private Map<String, String> storageLocations = new HashMap<>();
+ private Map<String, String> properties;
+ private AuditDTO audit;
+
+ private FilesetDTOBuilder() {}
+
+ /**
+ * Set the name of the fileset.
+ *
+ * @param name The name of the fileset.
+ * @return The builder instance.
+ */
+ public FilesetDTOBuilder name(String name) {
+ this.name = name;
+ return this;
+ }
+
+ /**
+ * Set the comment of the fileset.
+ *
+ * @param comment The comment of the fileset.
+ * @return The builder instance.
+ */
+ public FilesetDTOBuilder comment(String comment) {
+ this.comment = comment;
+ return this;
+ }
+
+ /**
+ * Set the type of the fileset.
+ *
+ * @param type The type of the fileset.
+ * @return The builder instance.
+ */
+ public FilesetDTOBuilder type(Type type) {
+ this.type = type;
+ return this;
+ }
+
+ /**
+ * Set the storage locations of the fileset.
+ *
+ * @param storageLocations The storage locations of the fileset.
+ * @return The builder instance.
+ */
+ public FilesetDTOBuilder storageLocations(Map<String, String>
storageLocations) {
+ this.storageLocations = ImmutableMap.copyOf(storageLocations);
+ return this;
+ }
+
+ /**
+ * Set the properties of the fileset.
+ *
+ * @param properties The properties of the fileset.
+ * @return The builder instance.
+ */
+ public FilesetDTOBuilder properties(Map<String, String> properties) {
+ this.properties = properties;
+ return this;
+ }
+
+ /**
+ * Set the audit information of the fileset.
+ *
+ * @param audit The audit information of the fileset.
+ * @return The builder instance.
+ */
+ public FilesetDTOBuilder audit(AuditDTO audit) {
+ this.audit = audit;
+ return this;
+ }
+
+ /**
+ * Build the FilesetDTO.
+ *
+ * @return The built FilesetDTO.
+ */
+ public FilesetDTO build() {
+ Preconditions.checkArgument(StringUtils.isNotBlank(name), "name cannot
be null or empty");
+ Preconditions.checkArgument(
+ !storageLocations.isEmpty(),
+ "storage locations cannot be empty. At least one location is
required.");
+ storageLocations.forEach(
+ (n, l) -> {
+ Preconditions.checkArgument(StringUtils.isNotBlank(n), "location
name cannot be empty");
+ Preconditions.checkArgument(
+ StringUtils.isNotBlank(l), "storage location cannot be empty");
+ });
+ return new FilesetDTO(
+ name,
+ comment,
+ type,
+ storageLocations.get(LOCATION_NAME_UNKNOWN),
+ storageLocations,
+ properties,
+ audit);
+ }
}
}
diff --git
a/common/src/main/java/org/apache/gravitino/dto/requests/FilesetCreateRequest.java
b/common/src/main/java/org/apache/gravitino/dto/requests/FilesetCreateRequest.java
index bd28189787..f4c79bca34 100644
---
a/common/src/main/java/org/apache/gravitino/dto/requests/FilesetCreateRequest.java
+++
b/common/src/main/java/org/apache/gravitino/dto/requests/FilesetCreateRequest.java
@@ -56,6 +56,10 @@ public class FilesetCreateRequest implements RESTRequest {
@JsonProperty("storageLocation")
private String storageLocation;
+ @Nullable
+ @JsonProperty("storageLocations")
+ private Map<String, String> storageLocations;
+
@Nullable
@JsonProperty("properties")
private Map<String, String> properties;
diff --git
a/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java
b/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java
index ce63398e60..5dd503ce84 100644
--- a/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java
+++ b/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java
@@ -612,7 +612,7 @@ public class DTOConverters {
.name(fileset.name())
.comment(fileset.comment())
.type(fileset.type())
- .storageLocation(fileset.storageLocation())
+ .storageLocations(fileset.storageLocations())
.properties(fileset.properties())
.audit(toDTO(fileset.auditInfo()))
.build();
diff --git
a/common/src/test/java/org/apache/gravitino/dto/file/TestFilesetDTO.java
b/common/src/test/java/org/apache/gravitino/dto/file/TestFilesetDTO.java
new file mode 100644
index 0000000000..eb43552a73
--- /dev/null
+++ b/common/src/test/java/org/apache/gravitino/dto/file/TestFilesetDTO.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.dto.file;
+
+import static org.apache.gravitino.file.Fileset.LOCATION_NAME_UNKNOWN;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.collect.ImmutableMap;
+import java.time.Instant;
+import java.util.Map;
+import org.apache.gravitino.dto.AuditDTO;
+import org.apache.gravitino.json.JsonUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestFilesetDTO {
+
+ @Test
+ public void testFilesetSerDe() throws JsonProcessingException {
+ AuditDTO audit =
AuditDTO.builder().withCreator("user1").withCreateTime(Instant.now()).build();
+ Map<String, String> props = ImmutableMap.of("key", "value");
+
+ // test with default location
+ FilesetDTO filesetDTO =
+ FilesetDTO.builder()
+ .name("fileset_test")
+ .comment("model comment")
+ .storageLocations(ImmutableMap.of(LOCATION_NAME_UNKNOWN, "/a/b/c",
"v1", "/d/e/f"))
+ .properties(props)
+ .audit(audit)
+ .build();
+ Assertions.assertEquals("fileset_test", filesetDTO.name());
+ Assertions.assertEquals("model comment", filesetDTO.comment());
+ Assertions.assertEquals(
+ ImmutableMap.of(LOCATION_NAME_UNKNOWN, "/a/b/c", "v1", "/d/e/f"),
+ filesetDTO.storageLocations());
+ Assertions.assertEquals("/a/b/c", filesetDTO.storageLocation());
+ Assertions.assertEquals(props, filesetDTO.properties());
+ Assertions.assertEquals(audit, filesetDTO.auditInfo());
+
+ String serJson = JsonUtils.objectMapper().writeValueAsString(filesetDTO);
+ FilesetDTO deserFilesetDTO = JsonUtils.objectMapper().readValue(serJson,
FilesetDTO.class);
+ Assertions.assertEquals(filesetDTO, deserFilesetDTO);
+
+ // test without default location exception
+ Exception exception =
+ Assertions.assertThrows(
+ IllegalArgumentException.class, () ->
FilesetDTO.builder().name("test").build());
+ Assertions.assertEquals(
+ "storage locations cannot be empty. At least one location is
required.",
+ exception.getMessage());
+
+ // test empty location name exception
+ exception =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> FilesetDTO.builder().storageLocations(ImmutableMap.of("",
"/a/b/c")).build());
+ Assertions.assertEquals("name cannot be null or empty",
exception.getMessage());
+
+ // test empty default location
+ exception =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ FilesetDTO.builder()
+ .name("test")
+ .storageLocations(ImmutableMap.of(LOCATION_NAME_UNKNOWN,
""))
+ .build());
+ Assertions.assertEquals("storage location cannot be empty",
exception.getMessage());
+
+ // test empty location
+ exception =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ FilesetDTO.builder()
+ .name("test")
+ .storageLocations(ImmutableMap.of(LOCATION_NAME_UNKNOWN,
"/a/b/c", "v1", ""))
+ .build());
+ Assertions.assertEquals("storage location cannot be empty",
exception.getMessage());
+ }
+}
diff --git a/docs/open-api/filesets.yaml b/docs/open-api/filesets.yaml
index f2b3fe8359..db5c5b190d 100644
--- a/docs/open-api/filesets.yaml
+++ b/docs/open-api/filesets.yaml
@@ -177,6 +177,13 @@ paths:
schema:
type: string
description: The sub path to the file or directory
+ - name: location_name
+ in: query
+ required: false
+ schema:
+ type: string
+ default: "default"
+ description: The location name in the fileset
responses:
"200":
$ref: "#/components/responses/FileLocationResponse"
@@ -249,6 +256,14 @@ components:
description: The location of the fileset. If the storage location of
managed fileset is empty, it will \
use the location of namespace. The storage location of external
fileset must be set.
nullable: true
+ storageLocations:
+ type: object
+ description: The storage locations of the fileset. If the storage
location of managed fileset is empty, \
+ it will use the location of namespace. The storage locations of
external fileset must be set.
+ nullable: true
+ default: { }
+ additionalProperties:
+ type: string
properties:
type: object
description: The properties of the fileset. Can be empty.
diff --git
a/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java
b/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java
index 713ba0c8eb..06342d4131 100644
---
a/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java
+++
b/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java
@@ -18,8 +18,13 @@
*/
package org.apache.gravitino.server.web.rest;
+import static org.apache.gravitino.file.Fileset.LOCATION_NAME_UNKNOWN;
+
import com.codahale.metrics.annotation.ResponseMetered;
import com.codahale.metrics.annotation.Timed;
+import com.google.common.collect.ImmutableMap;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import javax.inject.Inject;
@@ -124,12 +129,21 @@ public class FilesetOperations {
NameIdentifier ident =
NameIdentifierUtil.ofFileset(metalake, catalog, schema,
request.getName());
+ // set storageLocation value as unnamed location if provided
+ Map<String, String> tmpLocations =
+ new HashMap<>(
+ Optional.ofNullable(request.getStorageLocations())
+ .orElse(Collections.emptyMap()));
+ Optional.ofNullable(request.getStorageLocation())
+ .ifPresent(loc -> tmpLocations.put(LOCATION_NAME_UNKNOWN,
loc));
+ Map<String, String> storageLocations =
ImmutableMap.copyOf(tmpLocations);
+
Fileset fileset =
- dispatcher.createFileset(
+ dispatcher.createMultipleLocationFileset(
ident,
request.getComment(),
Optional.ofNullable(request.getType()).orElse(Fileset.Type.MANAGED),
- request.getStorageLocation(),
+ storageLocations,
request.getProperties());
Response response = Utils.ok(new
FilesetResponse(DTOConverters.toDTO(fileset)));
LOG.info("Fileset created: {}.{}.{}.{}", metalake, catalog,
schema, request.getName());
@@ -242,14 +256,16 @@ public class FilesetOperations {
@PathParam("catalog") String catalog,
@PathParam("schema") String schema,
@PathParam("fileset") String fileset,
- @QueryParam("sub_path") @NotNull String subPath) {
+ @QueryParam("sub_path") @NotNull String subPath,
+ @QueryParam("location_name") String locationName) {
LOG.info(
- "Received get file location request: {}.{}.{}.{}, sub path:{}",
+ "Received get file location request: {}.{}.{}.{}, sub path:{},
location name:{}",
metalake,
catalog,
schema,
fileset,
- RESTUtils.decodeString(subPath));
+ RESTUtils.decodeString(subPath),
+
Optional.ofNullable(locationName).map(RESTUtils::decodeString).orElse(null));
try {
return Utils.doAs(
httpRequest,
@@ -263,7 +279,10 @@ public class FilesetOperations {
CallerContext.CallerContextHolder.set(context);
}
String actualFileLocation =
- dispatcher.getFileLocation(ident,
RESTUtils.decodeString(subPath));
+ dispatcher.getFileLocation(
+ ident,
+ RESTUtils.decodeString(subPath),
+
Optional.ofNullable(locationName).map(RESTUtils::decodeString).orElse(null));
return Utils.ok(new FileLocationResponse(actualFileLocation));
});
} catch (Exception e) {
diff --git
a/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java
b/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java
index 4258346e49..f8248bc1d8 100644
---
a/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java
+++
b/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java
@@ -21,6 +21,7 @@ package org.apache.gravitino.server.web.rest;
import static org.apache.gravitino.Configs.TREE_LOCK_CLEAN_INTERVAL;
import static org.apache.gravitino.Configs.TREE_LOCK_MAX_NODE_IN_MEMORY;
import static org.apache.gravitino.Configs.TREE_LOCK_MIN_NODE_IN_MEMORY;
+import static org.apache.gravitino.file.Fileset.LOCATION_NAME_UNKNOWN;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
@@ -60,6 +61,7 @@ import
org.apache.gravitino.dto.responses.FileLocationResponse;
import org.apache.gravitino.dto.responses.FilesetResponse;
import org.apache.gravitino.exceptions.FilesetAlreadyExistsException;
import org.apache.gravitino.exceptions.NoSuchFilesetException;
+import org.apache.gravitino.exceptions.NoSuchLocationNameException;
import org.apache.gravitino.exceptions.NoSuchSchemaException;
import org.apache.gravitino.file.Fileset;
import org.apache.gravitino.file.FilesetChange;
@@ -204,6 +206,8 @@ public class TestFilesetOperations extends JerseyTest {
Assertions.assertEquals(fileset.type(), filesetDTO.type());
Assertions.assertEquals(fileset.comment(), filesetDTO.comment());
Assertions.assertEquals(fileset.properties(), filesetDTO.properties());
+ Assertions.assertEquals(fileset.storageLocation(),
filesetDTO.storageLocation());
+ Assertions.assertEquals(fileset.storageLocations(),
filesetDTO.storageLocations());
// Test throw NoSuchFilesetException
doThrow(new NoSuchFilesetException("no
found")).when(dispatcher).loadFileset(any());
@@ -242,7 +246,8 @@ public class TestFilesetOperations extends JerseyTest {
"mock comment",
"mock location",
ImmutableMap.of("k1", "v1"));
- when(dispatcher.createFileset(any(), any(), any(), any(),
any())).thenReturn(fileset);
+ when(dispatcher.createMultipleLocationFileset(any(), any(), any(), any(),
any()))
+ .thenReturn(fileset);
FilesetCreateRequest req =
FilesetCreateRequest.builder()
@@ -268,12 +273,59 @@ public class TestFilesetOperations extends JerseyTest {
Assertions.assertEquals("mock comment", filesetDTO.comment());
Assertions.assertEquals(Fileset.Type.MANAGED, filesetDTO.type());
Assertions.assertEquals("mock location", filesetDTO.storageLocation());
+ Assertions.assertEquals(
+ ImmutableMap.of(LOCATION_NAME_UNKNOWN, "mock location"),
filesetDTO.storageLocations());
Assertions.assertEquals(ImmutableMap.of("k1", "v1"),
filesetDTO.properties());
+ // test multiple locations
+ Map<String, String> locations =
+ ImmutableMap.of(
+ LOCATION_NAME_UNKNOWN,
+ "mock default",
+ "location1",
+ "mock location1",
+ "location2",
+ "mock location2");
+ fileset =
+ mockMultipleLocationsFileset(
+ "fileset1",
+ Fileset.Type.MANAGED,
+ "mock comment",
+ locations,
+ ImmutableMap.of("k1", "v1"));
+ when(dispatcher.createMultipleLocationFileset(any(), any(), any(), any(),
any()))
+ .thenReturn(fileset);
+
+ req =
+ FilesetCreateRequest.builder()
+ .name("fileset1")
+ .comment("mock comment")
+ .storageLocations(locations)
+ .properties(ImmutableMap.of("k1", "v1"))
+ .build();
+
+ resp =
+ target(filesetPath(metalake, catalog, schema))
+ .request(MediaType.APPLICATION_JSON_TYPE)
+ .accept("application/vnd.gravitino.v1+json")
+ .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE));
+
+ Assertions.assertEquals(Response.Status.OK.getStatusCode(),
resp.getStatus());
+
+ filesetResp = resp.readEntity(FilesetResponse.class);
+ Assertions.assertEquals(0, filesetResp.getCode());
+
+ filesetDTO = filesetResp.getFileset();
+ Assertions.assertEquals("fileset1", filesetDTO.name());
+ Assertions.assertEquals("mock comment", filesetDTO.comment());
+ Assertions.assertEquals(Fileset.Type.MANAGED, filesetDTO.type());
+ Assertions.assertEquals("mock default", filesetDTO.storageLocation());
+ Assertions.assertEquals(locations, filesetDTO.storageLocations());
+
// Test throw NoSuchSchemaException
doThrow(new NoSuchSchemaException("mock error"))
.when(dispatcher)
- .createFileset(any(), any(), any(), any(), any());
+ .createMultipleLocationFileset(any(), any(), any(), any(), any());
Response resp1 =
target(filesetPath(metalake, catalog, schema))
@@ -290,7 +342,7 @@ public class TestFilesetOperations extends JerseyTest {
// Test throw FilesetAlreadyExistsException
doThrow(new FilesetAlreadyExistsException("mock error"))
.when(dispatcher)
- .createFileset(any(), any(), any(), any(), any());
+ .createMultipleLocationFileset(any(), any(), any(), any(), any());
Response resp2 =
target(filesetPath(metalake, catalog, schema))
@@ -308,7 +360,7 @@ public class TestFilesetOperations extends JerseyTest {
// Test throw RuntimeException
doThrow(new RuntimeException("mock error"))
.when(dispatcher)
- .createFileset(any(), any(), any(), any(), any());
+ .createMultipleLocationFileset(any(), any(), any(), any(), any());
Response resp3 =
target(filesetPath(metalake, catalog, schema))
@@ -444,7 +496,7 @@ public class TestFilesetOperations extends JerseyTest {
// Test encoded subPath
NameIdentifier fullIdentifier = NameIdentifier.of(metalake, catalog,
schema, "fileset1");
String subPath = "/test/1";
- when(dispatcher.getFileLocation(fullIdentifier,
subPath)).thenReturn(subPath);
+ when(dispatcher.getFileLocation(fullIdentifier, subPath,
null)).thenReturn(subPath);
Response resp =
target(filesetPath(metalake, catalog, schema) + "fileset1/location")
.queryParam("sub_path", RESTUtils.encodeString(subPath))
@@ -458,10 +510,27 @@ public class TestFilesetOperations extends JerseyTest {
Assertions.assertEquals(subPath, contextResponse.getFileLocation());
+ // test specify location name
+ String locationName = "location1";
+ String location1 = "/test/location1";
+ when(dispatcher.getFileLocation(fullIdentifier, subPath,
locationName)).thenReturn(location1);
+ resp =
+ target(filesetPath(metalake, catalog, schema) + "fileset1/location")
+ .queryParam("sub_path", RESTUtils.encodeString(subPath))
+ .queryParam("location_name", locationName)
+ .request(MediaType.APPLICATION_JSON_TYPE)
+ .accept("application/vnd.gravitino.v1+json")
+ .get();
+ Assertions.assertEquals(Response.Status.OK.getStatusCode(),
resp.getStatus());
+
+ contextResponse = resp.readEntity(FileLocationResponse.class);
+ Assertions.assertEquals(0, contextResponse.getCode());
+ Assertions.assertEquals(location1, contextResponse.getFileLocation());
+
// Test throw NoSuchFilesetException
doThrow(new NoSuchFilesetException("no found"))
.when(dispatcher)
- .getFileLocation(fullIdentifier, subPath);
+ .getFileLocation(fullIdentifier, subPath, null);
Response resp1 =
target(filesetPath(metalake, catalog, schema) + "fileset1/location")
.queryParam("sub_path", RESTUtils.encodeString(subPath))
@@ -477,7 +546,7 @@ public class TestFilesetOperations extends JerseyTest {
// Test throw RuntimeException
doThrow(new RuntimeException("internal error"))
.when(dispatcher)
- .getFileLocation(fullIdentifier, subPath);
+ .getFileLocation(fullIdentifier, subPath, null);
Response resp2 =
target(filesetPath(metalake, catalog, schema) + "fileset1/location")
.queryParam("sub_path", RESTUtils.encodeString(subPath))
@@ -494,7 +563,7 @@ public class TestFilesetOperations extends JerseyTest {
// Test not encoded subPath
NameIdentifier fullIdentifier1 = NameIdentifier.of(metalake, catalog,
schema, "fileset2");
String subPath1 = "/test/2";
- when(dispatcher.getFileLocation(fullIdentifier1,
subPath1)).thenReturn(subPath1);
+ when(dispatcher.getFileLocation(fullIdentifier1, subPath1,
null)).thenReturn(subPath1);
Response resp3 =
target(filesetPath(metalake, catalog, schema) + "fileset2/location")
.queryParam("sub_path", subPath1)
@@ -508,12 +577,30 @@ public class TestFilesetOperations extends JerseyTest {
Assertions.assertEquals(subPath1, contextResponse1.getFileLocation());
+ // test throw NoSuchLocationNameException
+ doThrow(new NoSuchLocationNameException("no found"))
+ .when(dispatcher)
+ .getFileLocation(fullIdentifier, subPath, "not_exist");
+ Response noNameResp =
+ target(filesetPath(metalake, catalog, schema) + "fileset1/location")
+ .queryParam("sub_path", RESTUtils.encodeString(subPath))
+ .queryParam("location_name", "not_exist")
+ .request(MediaType.APPLICATION_JSON_TYPE)
+ .accept("application/vnd.gravitino.v1+json")
+ .get();
+ Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(),
noNameResp.getStatus());
+
+ ErrorResponse errorResp4 = noNameResp.readEntity(ErrorResponse.class);
+ Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE,
errorResp4.getCode());
+ Assertions.assertEquals(
+ NoSuchLocationNameException.class.getSimpleName(),
errorResp4.getType());
+
// Test header to caller context
try {
Map<String, String> callerContextMap = Maps.newHashMap();
NameIdentifier fullIdentifier2 = NameIdentifier.of(metalake, catalog,
schema, "fileset3");
String subPath2 = "/test/3";
- when(dispatcher.getFileLocation(fullIdentifier2, subPath2))
+ when(dispatcher.getFileLocation(fullIdentifier2, subPath2, null))
.thenAnswer(
(Answer<String>)
invocation -> {
@@ -561,10 +648,10 @@ public class TestFilesetOperations extends JerseyTest {
CallerContext context =
CallerContext.builder().withContext(testContextMap).build();
CallerContext.CallerContextHolder.set(context);
FilesetOperations mockOperations = Mockito.mock(FilesetOperations.class);
- Mockito.when(mockOperations.getFileLocation(any(), any(), any(), any(),
any()))
+ Mockito.when(mockOperations.getFileLocation(any(), any(), any(), any(),
any(), any()))
.thenCallRealMethod();
mockOperations.getFileLocation(
- "test_metalake", "test_catalog", "test_schema", "fileset4", "/test");
+ "test_metalake", "test_catalog", "test_schema", "fileset4", "/test",
"default");
Assertions.assertNull(CallerContext.CallerContextHolder.get());
}
@@ -610,7 +697,31 @@ public class TestFilesetOperations extends JerseyTest {
when(mockFileset.name()).thenReturn(filesetName);
when(mockFileset.type()).thenReturn(type);
when(mockFileset.comment()).thenReturn(comment);
- when(mockFileset.storageLocation()).thenReturn(storageLocation);
+ when(mockFileset.storageLocation()).thenCallRealMethod();
+ when(mockFileset.storageLocations())
+ .thenReturn(ImmutableMap.of(LOCATION_NAME_UNKNOWN, storageLocation));
+ when(mockFileset.properties()).thenReturn(properties);
+
+ Audit mockAudit = mock(Audit.class);
+ when(mockAudit.creator()).thenReturn("gravitino");
+ when(mockAudit.createTime()).thenReturn(Instant.now());
+ when(mockFileset.auditInfo()).thenReturn(mockAudit);
+
+ return mockFileset;
+ }
+
+ private static Fileset mockMultipleLocationsFileset(
+ String filesetName,
+ Fileset.Type type,
+ String comment,
+ Map<String, String> storageLocations,
+ Map<String, String> properties) {
+ Fileset mockFileset = mock(Fileset.class);
+ when(mockFileset.name()).thenReturn(filesetName);
+ when(mockFileset.type()).thenReturn(type);
+ when(mockFileset.comment()).thenReturn(comment);
+ when(mockFileset.storageLocation()).thenCallRealMethod();
+ when(mockFileset.storageLocations()).thenReturn(storageLocations);
when(mockFileset.properties()).thenReturn(properties);
Audit mockAudit = mock(Audit.class);
