This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new f574743abc Core: Track metadata file location in ViewMetadata (#8608)
f574743abc is described below
commit f574743abc8ca62e25c7c02f2c4d54f435b3238c
Author: Eduard Tudenhoefner <[email protected]>
AuthorDate: Thu Sep 21 21:44:10 2023 +0200
Core: Track metadata file location in ViewMetadata (#8608)
---
.../java/org/apache/iceberg/view/ViewMetadata.java | 21 ++++++-
.../apache/iceberg/view/ViewMetadataParser.java | 13 ++++-
.../org/apache/iceberg/view/TestViewMetadata.java | 38 +++++++++++++
.../iceberg/view/TestViewMetadataParser.java | 65 ++++++++++++++++++++++
4 files changed, 134 insertions(+), 3 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java
b/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java
index fe2f2c8b68..cb905bce09 100644
--- a/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java
+++ b/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java
@@ -26,6 +26,7 @@ import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import javax.annotation.Nullable;
import org.apache.iceberg.MetadataUpdate;
import org.apache.iceberg.Schema;
import org.apache.iceberg.exceptions.ValidationException;
@@ -78,6 +79,9 @@ public interface ViewMetadata extends Serializable {
List<MetadataUpdate> changes();
+ @Nullable
+ String metadataFileLocation();
+
default ViewVersion version(int versionId) {
return versionsById().get(versionId);
}
@@ -145,6 +149,7 @@ public interface ViewMetadata extends Serializable {
private int currentVersionId;
private String location;
private String uuid;
+ private String metadataLocation;
// internal change tracking
private Integer lastAddedVersionId = null;
@@ -176,6 +181,7 @@ public interface ViewMetadata extends Serializable {
this.currentVersionId = base.currentVersionId();
this.location = base.location();
this.uuid = base.uuid();
+ this.metadataLocation = null;
}
public Builder upgradeFormatVersion(int newFormatVersion) {
@@ -205,6 +211,11 @@ public interface ViewMetadata extends Serializable {
return this;
}
+ public Builder setMetadataLocation(String newMetadataLocation) {
+ this.metadataLocation = newMetadataLocation;
+ return this;
+ }
+
public Builder setCurrentVersionId(int newVersionId) {
if (newVersionId == LAST_ADDED) {
ValidationException.check(
@@ -375,6 +386,13 @@ public interface ViewMetadata extends Serializable {
Preconditions.checkArgument(null != location, "Invalid location: null");
Preconditions.checkArgument(versions.size() > 0, "Invalid view: no
versions were added");
+ // when associated with a metadata file, metadata must have no changes
so that the metadata
+ // matches exactly what is in the metadata file, which does not store
changes. metadata
+ // location with changes is inconsistent.
+ Preconditions.checkArgument(
+ metadataLocation == null || changes.isEmpty(),
+ "Cannot create view metadata with a metadata location and changes");
+
int historySize =
PropertyUtil.propertyAsInt(
properties,
@@ -412,7 +430,8 @@ public interface ViewMetadata extends Serializable {
retainedVersions,
retainedHistory,
properties,
- changes);
+ changes,
+ metadataLocation);
}
static List<ViewVersion> expireVersions(
diff --git a/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java
b/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java
index 0852db6a51..7a29c87bad 100644
--- a/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java
+++ b/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java
@@ -90,12 +90,20 @@ public class ViewMetadataParser {
gen.writeEndObject();
}
+ public static ViewMetadata fromJson(String metadataLocation, String json) {
+ return JsonUtil.parse(json, node ->
ViewMetadataParser.fromJson(metadataLocation, node));
+ }
+
public static ViewMetadata fromJson(String json) {
Preconditions.checkArgument(json != null, "Cannot parse view metadata from
null string");
return JsonUtil.parse(json, ViewMetadataParser::fromJson);
}
public static ViewMetadata fromJson(JsonNode json) {
+ return fromJson(null, json);
+ }
+
+ public static ViewMetadata fromJson(String metadataLocation, JsonNode json) {
Preconditions.checkArgument(json != null, "Cannot parse view metadata from
null object");
Preconditions.checkArgument(
json.isObject(), "Cannot parse view metadata from non-object: %s",
json);
@@ -142,7 +150,8 @@ public class ViewMetadataParser {
versions,
historyEntries,
properties,
- ImmutableList.of());
+ ImmutableList.of(),
+ metadataLocation);
}
public static void overwrite(ViewMetadata metadata, OutputFile outputFile) {
@@ -155,7 +164,7 @@ public class ViewMetadataParser {
public static ViewMetadata read(InputFile file) {
try (InputStream is = file.newStream()) {
- return fromJson(JsonUtil.mapper().readValue(is, JsonNode.class));
+ return fromJson(file.location(), JsonUtil.mapper().readValue(is,
JsonNode.class));
} catch (IOException e) {
throw new UncheckedIOException(String.format("Failed to read json file:
%s", file), e);
}
diff --git a/core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java
b/core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java
index acb344ffab..b525068cdf 100644
--- a/core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java
+++ b/core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java
@@ -522,4 +522,42 @@ public class TestViewMetadata {
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot reassign uuid");
}
+
+ @Test
+ public void viewMetadataWithMetadataLocation() {
+ Schema schema = new Schema(1, Types.NestedField.required(1, "x",
Types.LongType.get()));
+ ViewVersion viewVersion =
+ ImmutableViewVersion.builder()
+ .schemaId(schema.schemaId())
+ .versionId(1)
+ .timestampMillis(23L)
+ .putSummary("operation", "a")
+ .defaultNamespace(Namespace.of("ns"))
+ .build();
+
+ assertThatThrownBy(
+ () ->
+ ViewMetadata.builder()
+ .setLocation("custom-location")
+ .setMetadataLocation("metadata-location")
+ .addSchema(schema)
+ .addVersion(viewVersion)
+ .setCurrentVersionId(1)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Cannot create view metadata with a metadata location and
changes");
+
+ // setting metadata location without changes is ok
+ ViewMetadata viewMetadata =
+ ViewMetadata.buildFrom(
+ ViewMetadata.builder()
+ .setLocation("custom-location")
+ .addSchema(schema)
+ .addVersion(viewVersion)
+ .setCurrentVersionId(1)
+ .build())
+ .setMetadataLocation("metadata-location")
+ .build();
+
assertThat(viewMetadata.metadataFileLocation()).isEqualTo("metadata-location");
+ }
}
diff --git
a/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java
b/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java
index 5efbcf026f..076626d9fa 100644
--- a/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java
+++ b/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java
@@ -162,4 +162,69 @@ public class TestViewMetadataParser {
Path path =
Paths.get(getClass().getClassLoader().getResource(fileName).toURI());
return String.join("", java.nio.file.Files.readAllLines(path));
}
+
+ @Test
+ public void viewMetadataWithMetadataLocation() throws Exception {
+ ViewVersion version1 =
+ ImmutableViewVersion.builder()
+ .versionId(1)
+ .timestampMillis(4353L)
+ .summary(ImmutableMap.of("operation", "create"))
+ .schemaId(1)
+ .defaultCatalog("some-catalog")
+ .defaultNamespace(Namespace.empty())
+ .addRepresentations(
+ ImmutableSQLViewRepresentation.builder()
+ .sql("select 'foo' foo")
+ .dialect("spark-sql")
+ .build())
+ .build();
+
+ ViewVersion version2 =
+ ImmutableViewVersion.builder()
+ .versionId(2)
+ .schemaId(1)
+ .timestampMillis(5555L)
+ .summary(ImmutableMap.of("operation", "replace"))
+ .defaultCatalog("some-catalog")
+ .defaultNamespace(Namespace.empty())
+ .addRepresentations(
+ ImmutableSQLViewRepresentation.builder()
+ .sql("select 1 id, 'abc' data")
+ .dialect("spark-sql")
+ .build())
+ .build();
+
+ String json =
readViewMetadataInputFile("org/apache/iceberg/view/ValidViewMetadata.json");
+ String metadataLocation =
"s3://bucket/test/location/metadata/v1.metadata.json";
+ ViewMetadata expectedViewMetadata =
+ ViewMetadata.buildFrom(
+ ViewMetadata.builder()
+ .assignUUID("fa6506c3-7681-40c8-86dc-e36561f83385")
+ .addSchema(TEST_SCHEMA)
+ .addVersion(version1)
+ .addVersion(version2)
+ .setLocation("s3://bucket/test/location")
+ .setProperties(ImmutableMap.of("some-key", "some-value"))
+ .setCurrentVersionId(2)
+ .upgradeFormatVersion(1)
+ .build())
+ .setMetadataLocation(metadataLocation)
+ .build();
+
+ ViewMetadata actual = ViewMetadataParser.fromJson(metadataLocation, json);
+ assertThat(actual)
+ .usingRecursiveComparison()
+ .ignoringFieldsOfTypes(Schema.class)
+ .isEqualTo(expectedViewMetadata);
+
+ actual =
+ ViewMetadataParser.fromJson(
+ metadataLocation, ViewMetadataParser.toJson(expectedViewMetadata));
+ assertThat(actual)
+ .usingRecursiveComparison()
+ .ignoringFieldsOfTypes(Schema.class)
+ .isEqualTo(expectedViewMetadata);
+ assertThat(actual.metadataFileLocation()).isEqualTo(metadataLocation);
+ }
}
