This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new d9eb937bfb Core: Add UUID to ViewMetadata (#8591)
d9eb937bfb is described below
commit d9eb937bfb6bfed98b0a7e64ef402792a1a2b777
Author: Eduard Tudenhoefner <[email protected]>
AuthorDate: Wed Sep 20 23:19:32 2023 +0200
Core: Add UUID to ViewMetadata (#8591)
---
.../java/org/apache/iceberg/MetadataUpdate.java | 5 ++
.../java/org/apache/iceberg/view/ViewMetadata.java | 19 ++++++
.../apache/iceberg/view/ViewMetadataParser.java | 4 ++
.../org/apache/iceberg/view/TestViewMetadata.java | 73 +++++++++++++++++++---
.../iceberg/view/TestViewMetadataParser.java | 1 +
.../org/apache/iceberg/view/ValidViewMetadata.json | 1 +
.../view/ViewMetadataInvalidCurrentSchema.json | 1 +
.../view/ViewMetadataInvalidCurrentVersion.json | 1 +
.../view/ViewMetadataMissingCurrentVersion.json | 1 +
.../iceberg/view/ViewMetadataMissingLocation.json | 1 +
format/view-spec.md | 3 +
11 files changed, 102 insertions(+), 8 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/MetadataUpdate.java
b/core/src/main/java/org/apache/iceberg/MetadataUpdate.java
index 2cf16bca6c..363aabbff2 100644
--- a/core/src/main/java/org/apache/iceberg/MetadataUpdate.java
+++ b/core/src/main/java/org/apache/iceberg/MetadataUpdate.java
@@ -53,6 +53,11 @@ public interface MetadataUpdate extends Serializable {
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.assignUUID(uuid);
}
+
+ @Override
+ public void applyTo(ViewMetadata.Builder metadataBuilder) {
+ metadataBuilder.assignUUID(uuid);
+ }
}
class UpgradeFormatVersion implements MetadataUpdate {
diff --git a/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java
b/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java
index d4df7169fd..fe2f2c8b68 100644
--- a/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java
+++ b/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java
@@ -23,6 +23,7 @@ import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.iceberg.MetadataUpdate;
@@ -46,6 +47,8 @@ public interface ViewMetadata extends Serializable {
int SUPPORTED_VIEW_FORMAT_VERSION = 1;
int DEFAULT_VIEW_FORMAT_VERSION = 1;
+ String uuid();
+
int formatVersion();
String location();
@@ -141,6 +144,7 @@ public interface ViewMetadata extends Serializable {
private int formatVersion = DEFAULT_VIEW_FORMAT_VERSION;
private int currentVersionId;
private String location;
+ private String uuid;
// internal change tracking
private Integer lastAddedVersionId = null;
@@ -157,6 +161,7 @@ public interface ViewMetadata extends Serializable {
this.history = Lists.newArrayList();
this.properties = Maps.newHashMap();
this.changes = Lists.newArrayList();
+ this.uuid = null;
}
private Builder(ViewMetadata base) {
@@ -170,6 +175,7 @@ public interface ViewMetadata extends Serializable {
this.formatVersion = base.formatVersion();
this.currentVersionId = base.currentVersionId();
this.location = base.location();
+ this.uuid = base.uuid();
}
public Builder upgradeFormatVersion(int newFormatVersion) {
@@ -353,6 +359,18 @@ public interface ViewMetadata extends Serializable {
return this;
}
+ public ViewMetadata.Builder assignUUID(String newUUID) {
+ Preconditions.checkArgument(newUUID != null, "Cannot set uuid to null");
+ Preconditions.checkArgument(uuid == null || newUUID.equals(uuid),
"Cannot reassign uuid");
+
+ if (!newUUID.equals(uuid)) {
+ this.uuid = newUUID;
+ changes.add(new MetadataUpdate.AssignUUID(uuid));
+ }
+
+ return this;
+ }
+
public ViewMetadata build() {
Preconditions.checkArgument(null != location, "Invalid location: null");
Preconditions.checkArgument(versions.size() > 0, "Invalid view: no
versions were added");
@@ -386,6 +404,7 @@ public interface ViewMetadata extends Serializable {
}
return ImmutableViewMetadata.of(
+ null == uuid ? UUID.randomUUID().toString() : uuid,
formatVersion,
location,
schemas,
diff --git a/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java
b/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java
index c994c82ea8..0852db6a51 100644
--- a/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java
+++ b/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java
@@ -39,6 +39,7 @@ import org.apache.iceberg.util.JsonUtil;
public class ViewMetadataParser {
+ static final String VIEW_UUID = "view-uuid";
static final String FORMAT_VERSION = "format-version";
static final String LOCATION = "location";
static final String CURRENT_VERSION_ID = "current-version-id";
@@ -62,6 +63,7 @@ public class ViewMetadataParser {
gen.writeStartObject();
+ gen.writeStringField(VIEW_UUID, metadata.uuid());
gen.writeNumberField(FORMAT_VERSION, metadata.formatVersion());
gen.writeStringField(LOCATION, metadata.location());
JsonUtil.writeStringMap(PROPERTIES, metadata.properties(), gen);
@@ -98,6 +100,7 @@ public class ViewMetadataParser {
Preconditions.checkArgument(
json.isObject(), "Cannot parse view metadata from non-object: %s",
json);
+ String uuid = JsonUtil.getString(VIEW_UUID, json);
int formatVersion = JsonUtil.getInt(FORMAT_VERSION, json);
String location = JsonUtil.getString(LOCATION, json);
Map<String, String> properties = JsonUtil.getStringMap(PROPERTIES, json);
@@ -131,6 +134,7 @@ public class ViewMetadataParser {
}
return ImmutableViewMetadata.of(
+ uuid,
formatVersion,
location,
schemas,
diff --git a/core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java
b/core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java
index a852a716d5..acb344ffab 100644
--- a/core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java
+++ b/core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java
@@ -24,6 +24,7 @@ import static
org.assertj.core.api.Assertions.assertThatThrownBy;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
import org.apache.iceberg.MetadataUpdate;
import org.apache.iceberg.Schema;
import org.apache.iceberg.catalog.Namespace;
@@ -43,6 +44,8 @@ public class TestViewMetadata {
.defaultCatalog("prod")
.defaultNamespace(Namespace.of("default"))
.summary(ImmutableMap.of("operation", "create"))
+ .addRepresentations(
+
ImmutableSQLViewRepresentation.builder().dialect("spark").sql(sql).build())
.schemaId(1)
.build();
}
@@ -101,6 +104,10 @@ public class TestViewMetadata {
() ->
ViewMetadata.builder().setLocation("location").setCurrentVersionId(1).build())
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot set current version to unknown version: 1");
+
+ assertThatThrownBy(() -> ViewMetadata.builder().assignUUID(null).build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Cannot set uuid to null");
}
@Test
@@ -380,8 +387,10 @@ public class TestViewMetadata {
.defaultNamespace(Namespace.of("ns"))
.build();
+ String uuid = "fa6506c3-7681-40c8-86dc-e36561f83385";
ViewMetadata viewMetadata =
ViewMetadata.builder()
+ .assignUUID(uuid)
.setLocation("custom-location")
.setProperties(properties)
.addSchema(schemaOne)
@@ -406,23 +415,30 @@ public class TestViewMetadata {
assertThat(viewMetadata.properties()).isEqualTo(properties);
List<MetadataUpdate> changes = viewMetadata.changes();
- assertThat(changes).hasSize(8);
+ assertThat(changes).hasSize(9);
assertThat(changes)
.element(0)
+ .isInstanceOf(MetadataUpdate.AssignUUID.class)
+
.asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AssignUUID.class))
+ .extracting(MetadataUpdate.AssignUUID::uuid)
+ .isEqualTo(uuid);
+
+ assertThat(changes)
+ .element(1)
.isInstanceOf(MetadataUpdate.SetLocation.class)
.asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.SetLocation.class))
.extracting(MetadataUpdate.SetLocation::location)
.isEqualTo("custom-location");
assertThat(changes)
- .element(1)
+ .element(2)
.isInstanceOf(MetadataUpdate.SetProperties.class)
.asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.SetProperties.class))
.extracting(MetadataUpdate.SetProperties::updated)
.isEqualTo(properties);
assertThat(changes)
- .element(2)
+ .element(3)
.isInstanceOf(MetadataUpdate.AddSchema.class)
.asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddSchema.class))
.extracting(MetadataUpdate.AddSchema::schema)
@@ -430,7 +446,7 @@ public class TestViewMetadata {
.isEqualTo(1);
assertThat(changes)
- .element(3)
+ .element(4)
.isInstanceOf(MetadataUpdate.AddSchema.class)
.asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddSchema.class))
.extracting(MetadataUpdate.AddSchema::schema)
@@ -438,31 +454,72 @@ public class TestViewMetadata {
.isEqualTo(2);
assertThat(changes)
- .element(4)
+ .element(5)
.isInstanceOf(MetadataUpdate.AddViewVersion.class)
.asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddViewVersion.class))
.extracting(MetadataUpdate.AddViewVersion::viewVersion)
.isEqualTo(viewVersionOne);
assertThat(changes)
- .element(5)
+ .element(6)
.isInstanceOf(MetadataUpdate.AddViewVersion.class)
.asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddViewVersion.class))
.extracting(MetadataUpdate.AddViewVersion::viewVersion)
.isEqualTo(viewVersionTwo);
assertThat(changes)
- .element(6)
+ .element(7)
.isInstanceOf(MetadataUpdate.AddViewVersion.class)
.asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddViewVersion.class))
.extracting(MetadataUpdate.AddViewVersion::viewVersion)
.isEqualTo(viewVersionThree);
assertThat(changes)
- .element(7)
+ .element(8)
.isInstanceOf(MetadataUpdate.SetCurrentViewVersion.class)
.asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.SetCurrentViewVersion.class))
.extracting(MetadataUpdate.SetCurrentViewVersion::versionId)
.isEqualTo(-1);
}
+
+ @Test
+ public void uuidAssignment() {
+ String uuid = "fa6506c3-7681-40c8-86dc-e36561f83385";
+ ViewMetadata viewMetadata =
+ ViewMetadata.builder()
+ .assignUUID(uuid)
+ .setLocation("custom-location")
+ .addSchema(new Schema(1, Types.NestedField.required(1, "x",
Types.LongType.get())))
+ .addVersion(
+ ImmutableViewVersion.builder()
+ .schemaId(1)
+ .versionId(1)
+ .timestampMillis(23L)
+ .putSummary("operation", "create")
+ .defaultNamespace(Namespace.of("ns"))
+ .build())
+ .setCurrentVersionId(1)
+ .build();
+
+ assertThat(viewMetadata.uuid()).isEqualTo(uuid);
+
+ // uuid should be carried over
+ ViewMetadata updated = ViewMetadata.buildFrom(viewMetadata).build();
+ assertThat(updated.uuid()).isEqualTo(uuid);
+ assertThat(updated.changes()).isEmpty();
+
+ // assigning the same uuid shouldn't fail and shouldn't cause any changes
+ updated = ViewMetadata.buildFrom(viewMetadata).assignUUID(uuid).build();
+ assertThat(updated.uuid()).isEqualTo(uuid);
+ assertThat(updated.changes()).isEmpty();
+
+ // can't reassign view uuid
+ assertThatThrownBy(
+ () ->
+ ViewMetadata.buildFrom(viewMetadata)
+ .assignUUID(UUID.randomUUID().toString())
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Cannot reassign uuid");
+ }
}
diff --git
a/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java
b/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java
index 807a1df9b1..5efbcf026f 100644
--- a/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java
+++ b/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java
@@ -89,6 +89,7 @@ public class TestViewMetadataParser {
String json =
readViewMetadataInputFile("org/apache/iceberg/view/ValidViewMetadata.json");
ViewMetadata expectedViewMetadata =
ViewMetadata.builder()
+ .assignUUID("fa6506c3-7681-40c8-86dc-e36561f83385")
.addSchema(TEST_SCHEMA)
.addVersion(version1)
.addVersion(version2)
diff --git
a/core/src/test/resources/org/apache/iceberg/view/ValidViewMetadata.json
b/core/src/test/resources/org/apache/iceberg/view/ValidViewMetadata.json
index 9c0ae0ecbe..4e29ed8702 100644
--- a/core/src/test/resources/org/apache/iceberg/view/ValidViewMetadata.json
+++ b/core/src/test/resources/org/apache/iceberg/view/ValidViewMetadata.json
@@ -1,4 +1,5 @@
{
+ "view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385",
"format-version": 1,
"location": "s3://bucket/test/location",
"properties": {"some-key": "some-value"},
diff --git
a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentSchema.json
b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentSchema.json
index b63c6a6285..e6bdff2aad 100644
---
a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentSchema.json
+++
b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentSchema.json
@@ -1,4 +1,5 @@
{
+ "view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385",
"format-version": 1,
"location": "s3://bucket/test/location",
"properties": {"some-key": "some-value"},
diff --git
a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentVersion.json
b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentVersion.json
index fbcb2c9a41..8db0359c0d 100644
---
a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentVersion.json
+++
b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentVersion.json
@@ -1,4 +1,5 @@
{
+ "view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385",
"format-version": 1,
"location": "s3://bucket/test/location",
"properties": {"some-key": "some-value"},
diff --git
a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentVersion.json
b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentVersion.json
index f09a7a4aa6..07febf71c9 100644
---
a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentVersion.json
+++
b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentVersion.json
@@ -1,4 +1,5 @@
{
+ "view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385",
"format-version": 1,
"location": "s3://bucket/test/location",
"properties": {"some-key": "some-value"},
diff --git
a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingLocation.json
b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingLocation.json
index d0fa7d9392..aa6d56ead3 100644
---
a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingLocation.json
+++
b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingLocation.json
@@ -1,4 +1,5 @@
{
+ "view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385",
"format-version": 1,
"properties": {"some-key": "some-value"},
"current-schema-id": 1,
diff --git a/format/view-spec.md b/format/view-spec.md
index d7e0b7b7a6..26313193af 100644
--- a/format/view-spec.md
+++ b/format/view-spec.md
@@ -58,6 +58,7 @@ The view version metadata file has the following fields:
| Requirement | Field name | Description |
|-------------|----------------------|-------------|
+| _required_ | `view-uuid` | A UUID that identifies the view,
generated when the view is created. Implementations must throw an exception if
a view's UUID does not match the expected UUID after refreshing metadata |
| _required_ | `format-version` | An integer version number for the view
format; must be 1 |
| _required_ | `location` | The view's base location; used to
create metadata file locations |
| _required_ | `schemas` | A list of known schemas |
@@ -192,6 +193,7 @@
s3://bucket/warehouse/default.db/event_agg/metadata/00001-(uuid).metadata.json
```
```
{
+ "view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385",
"format-version" : 1,
"location" : "s3://bucket/warehouse/default.db/event_agg",
"current-version-id" : 1,
@@ -259,6 +261,7 @@
s3://bucket/warehouse/default.db/event_agg/metadata/00002-(uuid).metadata.json
```
```
{
+ "view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385",
"format-version" : 1,
"location" : "s3://bucket/warehouse/default.db/event_agg",
"current-version-id" : 1,
