This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 16a2e68ae6 [core][python] Record writer build version in snapshots
(#9283)
16a2e68ae6 is described below
commit 16a2e68ae6414cc6a2e139d873058ee8d6fcdc90
Author: YeJunHao <[email protected]>
AuthorDate: Thu Aug 20 10:08:46 2026 +0800
[core][python] Record writer build version in snapshots (#9283)
---
.../src/main/java/org/apache/paimon/Snapshot.java | 20 +++
paimon-core/pom.xml | 42 +++++++
.../src/main/java/org/apache/paimon/Changelog.java | 3 +
.../apache/paimon/operation/CoreFullVersion.java | 57 +++++++++
.../paimon/operation/FileStoreCommitImpl.java | 4 +
.../apache/paimon/table/system/SnapshotsTable.java | 6 +-
.../src/main/java/org/apache/paimon/tag/Tag.java | 4 +
.../test/java/org/apache/paimon/SnapshotTest.java | 27 ++++
.../DataEvolutionRowIdReassignerTest.java | 1 +
.../paimon/catalog/RenamingSnapshotCommitTest.java | 1 +
.../paimon/operation/CoreFullVersionTest.java | 49 ++++++++
.../paimon/operation/ExpireSnapshotsTest.java | 1 +
.../paimon/operation/FileStoreCommitTest.java | 14 +++
.../operation/commit/ConflictDetectionTest.java | 1 +
.../source/snapshot/WatermarkTimeTravelTest.java | 1 +
.../paimon/table/system/SnapshotsTableTest.java | 3 +-
.../org/apache/paimon/tag/TagAutoManagerTest.java | 2 +
.../test/java/org/apache/paimon/tag/TagTest.java | 4 +
.../apache/paimon/utils/SnapshotManagerTest.java | 5 +
paimon-python/dev/check-licensing.sh | 1 +
paimon-python/pypaimon/build_info.py | 86 +++++++++++++
paimon-python/pypaimon/changelog/changelog.py | 1 +
paimon-python/pypaimon/snapshot/snapshot.py | 1 +
.../pypaimon/table/system/snapshots_table.py | 5 +
paimon-python/pypaimon/tag/tag.py | 2 +
paimon-python/pypaimon/tests/build_info_test.py | 137 +++++++++++++++++++++
.../pypaimon/tests/changelog_manager_test.py | 29 +++--
.../pypaimon/tests/system/snapshots_table_test.py | 5 +-
paimon-python/pypaimon/tests/tag_ttl_serde_test.py | 1 +
.../pypaimon/tests/write/table_write_test.py | 6 +-
paimon-python/pypaimon/write/file_store_commit.py | 2 +
paimon-python/setup.py | 87 ++++++++++++-
.../read/SparkDataEvolutionVectorReadTest.java | 1 +
33 files changed, 593 insertions(+), 16 deletions(-)
diff --git a/paimon-api/src/main/java/org/apache/paimon/Snapshot.java
b/paimon-api/src/main/java/org/apache/paimon/Snapshot.java
index 2f98783ecc..d9de932506 100644
--- a/paimon-api/src/main/java/org/apache/paimon/Snapshot.java
+++ b/paimon-api/src/main/java/org/apache/paimon/Snapshot.java
@@ -61,6 +61,7 @@ public class Snapshot implements Serializable {
protected static final String FIELD_CHANGELOG_MANIFEST_LIST_SIZE =
"changelogManifestListSize";
protected static final String FIELD_INDEX_MANIFEST = "indexManifest";
protected static final String FIELD_COMMIT_USER = "commitUser";
+ protected static final String FIELD_WRITER_VERSION = "writerVersion";
protected static final String FIELD_COMMIT_IDENTIFIER = "commitIdentifier";
protected static final String FIELD_COMMIT_KIND = "commitKind";
protected static final String FIELD_TIME_MILLIS = "timeMillis";
@@ -132,6 +133,13 @@ public class Snapshot implements Serializable {
@JsonProperty(FIELD_COMMIT_USER)
protected final String commitUser;
+ // Version of the Paimon writer which created this snapshot.
+ // Null for snapshots created before writer version was introduced.
+ @JsonProperty(FIELD_WRITER_VERSION)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @Nullable
+ protected final String writerVersion;
+
// Mainly for snapshot deduplication.
//
// If multiple snapshots have the same commitIdentifier, reading from any
of these snapshots
@@ -206,6 +214,7 @@ public class Snapshot implements Serializable {
@Nullable Long changelogManifestListSize,
@Nullable String indexManifest,
String commitUser,
+ @Nullable String writerVersion,
long commitIdentifier,
CommitKind commitKind,
long timeMillis,
@@ -230,6 +239,7 @@ public class Snapshot implements Serializable {
changelogManifestListSize,
indexManifest,
commitUser,
+ writerVersion,
commitIdentifier,
commitKind,
timeMillis,
@@ -258,6 +268,7 @@ public class Snapshot implements Serializable {
Long changelogManifestListSize,
@JsonProperty(FIELD_INDEX_MANIFEST) @Nullable String indexManifest,
@JsonProperty(FIELD_COMMIT_USER) String commitUser,
+ @JsonProperty(FIELD_WRITER_VERSION) @Nullable String writerVersion,
@JsonProperty(FIELD_COMMIT_IDENTIFIER) long commitIdentifier,
@JsonProperty(FIELD_COMMIT_KIND) CommitKind commitKind,
@JsonProperty(FIELD_TIME_MILLIS) long timeMillis,
@@ -281,6 +292,7 @@ public class Snapshot implements Serializable {
this.changelogManifestListSize = changelogManifestListSize;
this.indexManifest = indexManifest;
this.commitUser = commitUser;
+ this.writerVersion = writerVersion;
this.commitIdentifier = commitIdentifier;
this.commitKind = commitKind;
this.timeMillis = timeMillis;
@@ -360,6 +372,12 @@ public class Snapshot implements Serializable {
return commitUser;
}
+ @JsonGetter(FIELD_WRITER_VERSION)
+ @Nullable
+ public String writerVersion() {
+ return writerVersion;
+ }
+
@JsonGetter(FIELD_COMMIT_IDENTIFIER)
public long commitIdentifier() {
return commitIdentifier;
@@ -440,6 +458,7 @@ public class Snapshot implements Serializable {
changelogManifestListSize,
indexManifest,
commitUser,
+ writerVersion,
commitIdentifier,
commitKind,
timeMillis,
@@ -474,6 +493,7 @@ public class Snapshot implements Serializable {
&& Objects.equals(changelogManifestListSize,
that.changelogManifestListSize)
&& Objects.equals(indexManifest, that.indexManifest)
&& Objects.equals(commitUser, that.commitUser)
+ && Objects.equals(writerVersion, that.writerVersion)
&& commitIdentifier == that.commitIdentifier
&& commitKind == that.commitKind
&& timeMillis == that.timeMillis
diff --git a/paimon-core/pom.xml b/paimon-core/pom.xml
index adb4568229..010672bd32 100644
--- a/paimon-core/pom.xml
+++ b/paimon-core/pom.xml
@@ -275,6 +275,48 @@ under the License.
<build>
<plugins>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>generate-build-info</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <!-- Prevent Git from discovering an unrelated
repository
+ containing a Paimon source archive. -->
+ <property name="paimon.core.git.ceiling"
+ location="${project.basedir}/../.."/>
+
+ <exec executable="git"
+ dir="${project.basedir}"
+ failifexecutionfails="false"
+ failonerror="false"
+
outputproperty="paimon.core.git.commit.id"
+ resultproperty="paimon.core.git.result">
+ <env key="GIT_CEILING_DIRECTORIES"
+ value="${paimon.core.git.ceiling}"/>
+ <arg value="rev-parse"/>
+ <arg value="HEAD"/>
+ </exec>
+ <condition property="paimon.core.commit.id"
+
value="${paimon.core.git.commit.id}">
+ <equals arg1="${paimon.core.git.result}"
arg2="0"/>
+ </condition>
+ <!-- Git metadata is not available in source
archives. -->
+ <property name="paimon.core.commit.id"
value="UNKNOWN"/>
+ <mkdir
dir="${project.build.outputDirectory}/META-INF"/>
+ <echo
file="${project.build.outputDirectory}/META-INF/paimon-core.full-version"
+
message="java-${project.version}-${paimon.core.commit.id}"/>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
diff --git a/paimon-core/src/main/java/org/apache/paimon/Changelog.java
b/paimon-core/src/main/java/org/apache/paimon/Changelog.java
index aaa1f7f4d5..4433c80619 100644
--- a/paimon-core/src/main/java/org/apache/paimon/Changelog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/Changelog.java
@@ -55,6 +55,7 @@ public class Changelog extends Snapshot {
snapshot.changelogManifestListSize(),
snapshot.indexManifest(),
snapshot.commitUser(),
+ snapshot.writerVersion(),
snapshot.commitIdentifier(),
snapshot.commitKind(),
snapshot.timeMillis(),
@@ -83,6 +84,7 @@ public class Changelog extends Snapshot {
Long changelogManifestListSize,
@JsonProperty(FIELD_INDEX_MANIFEST) @Nullable String indexManifest,
@JsonProperty(FIELD_COMMIT_USER) String commitUser,
+ @JsonProperty(FIELD_WRITER_VERSION) @Nullable String writerVersion,
@JsonProperty(FIELD_COMMIT_IDENTIFIER) long commitIdentifier,
@JsonProperty(FIELD_COMMIT_KIND) CommitKind commitKind,
@JsonProperty(FIELD_TIME_MILLIS) long timeMillis,
@@ -107,6 +109,7 @@ public class Changelog extends Snapshot {
changelogManifestListSize,
indexManifest,
commitUser,
+ writerVersion,
commitIdentifier,
commitKind,
timeMillis,
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/CoreFullVersion.java
b/paimon-core/src/main/java/org/apache/paimon/operation/CoreFullVersion.java
new file mode 100644
index 0000000000..314e75c85e
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/CoreFullVersion.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.operation;
+
+import javax.annotation.Nullable;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+
+/** Loads the full version of the current Paimon Core build. */
+final class CoreFullVersion {
+
+ @Nullable private static final String FULL_VERSION = load();
+
+ private CoreFullVersion() {}
+
+ @Nullable
+ static String get() {
+ return FULL_VERSION;
+ }
+
+ @Nullable
+ private static String load() {
+ InputStream inputStream =
+
CoreFullVersion.class.getResourceAsStream("/META-INF/paimon-core.full-version");
+ if (inputStream == null) {
+ return null;
+ }
+
+ try (BufferedReader reader =
+ new BufferedReader(new InputStreamReader(inputStream,
StandardCharsets.UTF_8))) {
+ String fullVersion = reader.readLine();
+ return fullVersion == null || fullVersion.trim().isEmpty() ? null
: fullVersion.trim();
+ } catch (IOException e) {
+ return null;
+ }
+ }
+}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
index 505faba772..02ce5dc05d 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
@@ -1229,6 +1229,7 @@ public class FileStoreCommitImpl implements
FileStoreCommit {
changelogManifestList == null ? null :
changelogManifestList.getRight(),
indexManifest,
commitUser,
+ CoreFullVersion.get(),
identifier,
commitKind,
System.currentTimeMillis(),
@@ -1384,6 +1385,7 @@ public class FileStoreCommitImpl implements
FileStoreCommit {
null,
indexManifest,
commitUser,
+ CoreFullVersion.get(),
Long.MAX_VALUE,
CommitKind.OVERWRITE,
System.currentTimeMillis(),
@@ -1470,6 +1472,7 @@ public class FileStoreCommitImpl implements
FileStoreCommit {
null,
targetSnapshot.indexManifest(),
commitUser,
+ CoreFullVersion.get(),
Long.MAX_VALUE,
CommitKind.OVERWRITE,
System.currentTimeMillis(),
@@ -1618,6 +1621,7 @@ public class FileStoreCommitImpl implements
FileStoreCommit {
null,
latestSnapshot.indexManifest(),
commitUser,
+ CoreFullVersion.get(),
Long.MAX_VALUE,
CommitKind.COMPACT,
System.currentTimeMillis(),
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java
b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java
index 201192fde6..75d46b708e 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java
@@ -109,8 +109,9 @@ public class SnapshotsTable implements ReadonlyTable {
new DataField(11, "changelog_record_count", new
BigIntType(true)),
new DataField(12, "watermark", new
BigIntType(true)),
new DataField(13, "next_row_id", new
BigIntType(true)),
+ new DataField(14, "operation",
SerializationUtils.newStringType(true)),
new DataField(
- 14, "operation",
SerializationUtils.newStringType(true))));
+ 15, "writer_version",
SerializationUtils.newStringType(true))));
private final FileIO fileIO;
private final Path location;
@@ -344,7 +345,8 @@ public class SnapshotsTable implements ReadonlyTable {
snapshot.nextRowId(),
snapshot.operation() == null
? null
- :
BinaryString.fromString(snapshot.operation().toString()));
+ :
BinaryString.fromString(snapshot.operation().toString()),
+ BinaryString.fromString(snapshot.writerVersion()));
}
}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/tag/Tag.java
b/paimon-core/src/main/java/org/apache/paimon/tag/Tag.java
index 1acd425cd4..ac435782d0 100644
--- a/paimon-core/src/main/java/org/apache/paimon/tag/Tag.java
+++ b/paimon-core/src/main/java/org/apache/paimon/tag/Tag.java
@@ -71,6 +71,7 @@ public class Tag extends Snapshot {
Long changelogManifestListSize,
@JsonProperty(FIELD_INDEX_MANIFEST) @Nullable String indexManifest,
@JsonProperty(FIELD_COMMIT_USER) String commitUser,
+ @JsonProperty(FIELD_WRITER_VERSION) @Nullable String writerVersion,
@JsonProperty(FIELD_COMMIT_IDENTIFIER) long commitIdentifier,
@JsonProperty(FIELD_COMMIT_KIND) CommitKind commitKind,
@JsonProperty(FIELD_TIME_MILLIS) long timeMillis,
@@ -97,6 +98,7 @@ public class Tag extends Snapshot {
changelogManifestListSize,
indexManifest,
commitUser,
+ writerVersion,
commitIdentifier,
commitKind,
timeMillis,
@@ -137,6 +139,7 @@ public class Tag extends Snapshot {
snapshot.changelogManifestListSize(),
snapshot.indexManifest(),
snapshot.commitUser(),
+ snapshot.writerVersion(),
snapshot.commitIdentifier(),
snapshot.commitKind(),
snapshot.timeMillis(),
@@ -166,6 +169,7 @@ public class Tag extends Snapshot {
changelogManifestListSize,
indexManifest,
commitUser,
+ writerVersion,
commitIdentifier,
commitKind,
timeMillis,
diff --git a/paimon-core/src/test/java/org/apache/paimon/SnapshotTest.java
b/paimon-core/src/test/java/org/apache/paimon/SnapshotTest.java
index d67e9feb03..60e9efe983 100644
--- a/paimon-core/src/test/java/org/apache/paimon/SnapshotTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/SnapshotTest.java
@@ -68,6 +68,33 @@ public class SnapshotTest {
+ " \"unknownKey\" : 22222\n"
+ "}");
assertThat(snapshot.uuid()).isNull();
+ assertThat(snapshot.writerVersion()).isNull();
+ }
+
+ @Test
+ public void testSnapshotWriterVersion() {
+ String json =
+ "{\n"
+ + " \"version\" : 3,\n"
+ + " \"id\" : 1,\n"
+ + " \"schemaId\" : 0,\n"
+ + " \"baseManifestList\" : \"m-0\",\n"
+ + " \"deltaManifestList\" : \"m-1\",\n"
+ + " \"commitUser\" : \"user\",\n"
+ + " \"writerVersion\" :
\"java-2.1-SNAPSHOT-0123456789012345678901234567890123456789\",\n"
+ + " \"commitIdentifier\" : 0,\n"
+ + " \"commitKind\" : \"APPEND\",\n"
+ + " \"timeMillis\" : 1000,\n"
+ + " \"totalRecordCount\" : 10,\n"
+ + " \"deltaRecordCount\" : 5\n"
+ + "}";
+ Snapshot snapshot = Snapshot.fromJson(json);
+ assertThat(snapshot.writerVersion())
+
.isEqualTo("java-2.1-SNAPSHOT-0123456789012345678901234567890123456789");
+ assertThat(Snapshot.fromJson(snapshot.toJson())).isEqualTo(snapshot);
+ Changelog changelog = new Changelog(snapshot);
+
assertThat(changelog.writerVersion()).isEqualTo(snapshot.writerVersion());
+
assertThat(Changelog.fromJson(changelog.toJson())).isEqualTo(changelog);
}
@Test
diff --git
a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java
b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java
index 0de855be14..08b35b7bdf 100644
---
a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java
@@ -2896,6 +2896,7 @@ public class DataEvolutionRowIdReassignerTest extends
TableTestBase {
latest.changelogManifestListSize(),
indexManifest,
latest.commitUser(),
+ latest.writerVersion(),
latest.commitIdentifier(),
latest.commitKind(),
latest.timeMillis(),
diff --git
a/paimon-core/src/test/java/org/apache/paimon/catalog/RenamingSnapshotCommitTest.java
b/paimon-core/src/test/java/org/apache/paimon/catalog/RenamingSnapshotCommitTest.java
index 1eef5c53cf..4e7fa27e7b 100644
---
a/paimon-core/src/test/java/org/apache/paimon/catalog/RenamingSnapshotCommitTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/catalog/RenamingSnapshotCommitTest.java
@@ -128,6 +128,7 @@ public class RenamingSnapshotCommitTest {
changelogManifestListSize,
indexManifest,
commitUser,
+ null,
commitIdentifier,
commitKind,
timeMillis,
diff --git
a/paimon-core/src/test/java/org/apache/paimon/operation/CoreFullVersionTest.java
b/paimon-core/src/test/java/org/apache/paimon/operation/CoreFullVersionTest.java
new file mode 100644
index 0000000000..0ed7f2e5b4
--- /dev/null
+++
b/paimon-core/src/test/java/org/apache/paimon/operation/CoreFullVersionTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.operation;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link CoreFullVersion}. */
+public class CoreFullVersionTest {
+
+ @Test
+ public void testFullVersion() throws Exception {
+ String fullVersion =
readResource("/META-INF/paimon-core.full-version");
+
+ assertThat(CoreFullVersion.get()).isEqualTo(fullVersion);
+ assertThat(fullVersion).matches("java-.+-(UNKNOWN|[0-9a-f]{40})");
+ }
+
+ private String readResource(String path) throws Exception {
+ InputStream inputStream =
CoreFullVersion.class.getResourceAsStream(path);
+ assertThat(inputStream).isNotNull();
+ try (BufferedReader reader =
+ new BufferedReader(new InputStreamReader(inputStream,
StandardCharsets.UTF_8))) {
+ return reader.readLine().trim();
+ }
+ }
+}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java
b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java
index c4de0f44e1..143f8445b8 100644
---
a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java
@@ -411,6 +411,7 @@ public class ExpireSnapshotsTest {
null,
null,
"test",
+ null,
0L,
Snapshot.CommitKind.APPEND,
0L,
diff --git
a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
index 3297383dcb..c19f318073 100644
---
a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
@@ -1434,6 +1434,19 @@ public class FileStoreCommitTest {
}
}
+ @Test
+ public void testSnapshotWriterVersion() throws Exception {
+ TestFileStore store = createStore(false);
+
+ try (FileStoreCommit fileStoreCommit = store.newCommit()) {
+ fileStoreCommit.ignoreEmptyCommit(false);
+ fileStoreCommit.commit(new ManifestCommittable(0), false);
+ }
+
+
assertThat(checkNotNull(store.snapshotManager().latestSnapshot()).writerVersion())
+ .isEqualTo(CoreFullVersion.get());
+ }
+
@Test
public void testGlobalIndexCommitChecksExistingRowIds() throws Exception {
TestFileStore store = createRowTrackingDataEvolutionStore();
@@ -2192,6 +2205,7 @@ public class FileStoreCommitTest {
null,
previousSnapshot == null ? null :
previousSnapshot.indexManifest(),
"conflict-user",
+ snapshot.writerVersion(),
Long.MAX_VALUE,
Snapshot.CommitKind.ANALYZE,
System.currentTimeMillis(),
diff --git
a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java
b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java
index de0a04bb3d..1fa8430b66 100644
---
a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java
@@ -1540,6 +1540,7 @@ class ConflictDetectionTest {
null,
null,
"commit-user",
+ null,
id,
Snapshot.CommitKind.APPEND,
id,
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/WatermarkTimeTravelTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/WatermarkTimeTravelTest.java
index 569e2d5bd4..172069ede6 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/WatermarkTimeTravelTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/WatermarkTimeTravelTest.java
@@ -262,6 +262,7 @@ public class WatermarkTimeTravelTest extends
ScannerTestBase {
s.changelogManifestListSize(),
s.indexManifest(),
s.commitUser(),
+ s.writerVersion(),
s.commitIdentifier(),
s.commitKind(),
s.timeMillis(),
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java
index 34d634e642..560fa72de4 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java
@@ -161,7 +161,8 @@ public class SnapshotsTableTest extends TableTestBase {
snapshot.nextRowId(),
snapshot.operation() == null
? null
- :
BinaryString.fromString(snapshot.operation().toString())));
+ :
BinaryString.fromString(snapshot.operation().toString()),
+
BinaryString.fromString(snapshot.writerVersion())));
}
return expectedRow;
diff --git
a/paimon-core/src/test/java/org/apache/paimon/tag/TagAutoManagerTest.java
b/paimon-core/src/test/java/org/apache/paimon/tag/TagAutoManagerTest.java
index d262110f03..d7951fd584 100644
--- a/paimon-core/src/test/java/org/apache/paimon/tag/TagAutoManagerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/tag/TagAutoManagerTest.java
@@ -401,6 +401,7 @@ public class TagAutoManagerTest extends
PrimaryKeyTableTestBase {
null,
null,
null,
+ null,
0L,
Snapshot.CommitKind.APPEND,
1000,
@@ -431,6 +432,7 @@ public class TagAutoManagerTest extends
PrimaryKeyTableTestBase {
null,
null,
null,
+ null,
0L,
Snapshot.CommitKind.APPEND,
1000,
diff --git a/paimon-core/src/test/java/org/apache/paimon/tag/TagTest.java
b/paimon-core/src/test/java/org/apache/paimon/tag/TagTest.java
index 459e006f17..4a9a0c6eb5 100644
--- a/paimon-core/src/test/java/org/apache/paimon/tag/TagTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/tag/TagTest.java
@@ -44,6 +44,7 @@ public class TagTest {
null,
null,
null,
+ "java-2.1-SNAPSHOT-commit-id",
0L,
Snapshot.CommitKind.APPEND,
1000,
@@ -68,6 +69,7 @@ public class TagTest {
+ " \"baseManifestList\" : null,\n"
+ " \"deltaManifestList\" : null,\n"
+ " \"commitUser\" : null,\n"
+ + " \"writerVersion\" :
\"java-2.1-SNAPSHOT-commit-id\",\n"
+ " \"commitIdentifier\" : 0,\n"
+ " \"commitKind\" : \"APPEND\",\n"
+ " \"timeMillis\" : 1000,\n"
@@ -94,6 +96,7 @@ public class TagTest {
+ " \"baseManifestList\" : null,\n"
+ " \"deltaManifestList\" : null,\n"
+ " \"commitUser\" : null,\n"
+ + " \"writerVersion\" :
\"java-2.1-SNAPSHOT-commit-id\",\n"
+ " \"commitIdentifier\" : 0,\n"
+ " \"commitKind\" : \"APPEND\",\n"
+ " \"timeMillis\" : 1000,\n"
@@ -106,5 +109,6 @@ public class TagTest {
Tag newTag = Tag.fromJson(tagJson);
assertEquals(tag, newTag);
+ assertEquals("java-2.1-SNAPSHOT-commit-id",
newTag.trimToSnapshot().writerVersion());
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java
b/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java
index 2750feb902..5adbf4129f 100644
--- a/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java
@@ -428,6 +428,7 @@ public class SnapshotManagerTest {
null,
null,
null,
+ null,
0L,
Snapshot.CommitKind.APPEND,
millis,
@@ -453,6 +454,7 @@ public class SnapshotManagerTest {
null,
null,
null,
+ null,
0L,
Snapshot.CommitKind.APPEND,
millis,
@@ -487,6 +489,7 @@ public class SnapshotManagerTest {
null,
null,
null,
+ null,
0L,
Snapshot.CommitKind.APPEND,
millis,
@@ -519,6 +522,7 @@ public class SnapshotManagerTest {
null,
null,
"lastCommitUser",
+ null,
0L,
Snapshot.CommitKind.APPEND,
i * 1000,
@@ -573,6 +577,7 @@ public class SnapshotManagerTest {
null,
null,
null,
+ null,
0L,
Snapshot.CommitKind.APPEND,
i * 1000,
diff --git a/paimon-python/dev/check-licensing.sh
b/paimon-python/dev/check-licensing.sh
index ec1fe9df0e..c867a0d7b6 100755
--- a/paimon-python/dev/check-licensing.sh
+++ b/paimon-python/dev/check-licensing.sh
@@ -79,6 +79,7 @@ else
-e '.*entry_points[.]txt$'
-e '.*requires[.]txt$'
-e '.*top_level[.]txt$'
+ -e '.*_full_version$'
-e '.*[.]jsonl$'
-d "${PACKAGE_DIR}"
)
diff --git a/paimon-python/pypaimon/build_info.py
b/paimon-python/pypaimon/build_info.py
new file mode 100644
index 0000000000..e034f356fd
--- /dev/null
+++ b/paimon-python/pypaimon/build_info.py
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import re
+import subprocess
+
+_UNKNOWN = "UNKNOWN"
+_FULL_VERSION_FILE = os.path.join(os.path.dirname(__file__), "_full_version")
+_SETUP_FILE = os.path.join(os.path.dirname(os.path.dirname(__file__)),
"setup.py")
+
+
+def _repository_root():
+ python_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ parent = os.path.dirname(python_root)
+ if os.path.basename(python_root) == "paimon-python" and os.path.exists(
+ os.path.join(parent, "pom.xml")):
+ return parent
+ return python_root
+
+
+def git_commit_id():
+ """Return the current Paimon Git revision without discovering an outer
repo."""
+ repository_root = _repository_root()
+ env = os.environ.copy()
+ env["GIT_CEILING_DIRECTORIES"] = os.path.dirname(repository_root)
+ try:
+ return subprocess.check_output(
+ ["git", "-C", repository_root, "rev-parse", "HEAD"],
+ stderr=subprocess.DEVNULL,
+ env=env,
+ ).decode("utf-8").strip()
+ except Exception:
+ return _UNKNOWN
+
+
+def _source_version():
+ try:
+ with open(_SETUP_FILE, "r") as setup_file:
+ match = re.search(
+ r'^VERSION = ["\']([^"\']+)["\']$',
+ setup_file.read(),
+ re.MULTILINE,
+ )
+ return None if match is None else match.group(1)
+ except OSError:
+ return None
+
+
+def _load_full_version():
+ """Return the embedded full version, or derive it from the checkout."""
+ try:
+ with open(_FULL_VERSION_FILE, "r") as full_version_file:
+ value = full_version_file.read().strip()
+ if value:
+ return value
+ except OSError:
+ pass
+ version = _source_version()
+ return (
+ _UNKNOWN
+ if version is None
+ else "python-{}-{}".format(version, git_commit_id())
+ )
+
+
+_FULL_VERSION = _load_full_version()
+
+
+def full_version():
+ """Return ``<pypaimon-version>-<commit-id>`` for snapshot provenance."""
+ return _FULL_VERSION
diff --git a/paimon-python/pypaimon/changelog/changelog.py
b/paimon-python/pypaimon/changelog/changelog.py
index f562395390..195d530638 100644
--- a/paimon-python/pypaimon/changelog/changelog.py
+++ b/paimon-python/pypaimon/changelog/changelog.py
@@ -47,6 +47,7 @@ class Changelog(Snapshot):
changelog_manifest_list_size=snapshot.changelog_manifest_list_size,
index_manifest=snapshot.index_manifest,
commit_user=snapshot.commit_user,
+ writer_version=snapshot.writer_version,
commit_identifier=snapshot.commit_identifier,
commit_kind=snapshot.commit_kind,
time_millis=snapshot.time_millis,
diff --git a/paimon-python/pypaimon/snapshot/snapshot.py
b/paimon-python/pypaimon/snapshot/snapshot.py
index 407f27f0d7..d10e80a029 100644
--- a/paimon-python/pypaimon/snapshot/snapshot.py
+++ b/paimon-python/pypaimon/snapshot/snapshot.py
@@ -57,3 +57,4 @@ class Snapshot:
"json_missing_default": None,
},
)
+ writer_version: Optional[str] = optional_json_field("writerVersion",
"non_null")
diff --git a/paimon-python/pypaimon/table/system/snapshots_table.py
b/paimon-python/pypaimon/table/system/snapshots_table.py
index be7331d21f..d32c36163c 100644
--- a/paimon-python/pypaimon/table/system/snapshots_table.py
+++ b/paimon-python/pypaimon/table/system/snapshots_table.py
@@ -40,6 +40,8 @@ TABLE_TYPE = RowType(False, [
DataField(11, "changelog_record_count", AtomicType("BIGINT",
nullable=True)),
DataField(12, "watermark", AtomicType("BIGINT", nullable=True)),
DataField(13, "next_row_id", AtomicType("BIGINT", nullable=True)),
+ # Keep the field ID aligned with Java; ID 14 is reserved for operation.
+ DataField(15, "writer_version", AtomicType("STRING", nullable=True)),
])
@@ -75,6 +77,7 @@ class SnapshotsTable(SystemTable):
changelog_record_counts: List[Optional[int]] = []
watermarks: List[Optional[int]] = []
next_row_ids: List[Optional[int]] = []
+ writer_versions: List[Optional[str]] = []
for snap in snapshots:
snapshot_ids.append(int(snap.id))
@@ -99,6 +102,7 @@ class SnapshotsTable(SystemTable):
None if snap.watermark is None else int(snap.watermark))
next_row_ids.append(
None if snap.next_row_id is None else int(snap.next_row_id))
+ writer_versions.append(snap.writer_version)
return pyarrow.table({
"snapshot_id": pyarrow.array(snapshot_ids, type=pyarrow.int64()),
@@ -122,4 +126,5 @@ class SnapshotsTable(SystemTable):
changelog_record_counts, type=pyarrow.int64()),
"watermark": pyarrow.array(watermarks, type=pyarrow.int64()),
"next_row_id": pyarrow.array(next_row_ids, type=pyarrow.int64()),
+ "writer_version": pyarrow.array(writer_versions,
type=pyarrow.string()),
})
diff --git a/paimon-python/pypaimon/tag/tag.py
b/paimon-python/pypaimon/tag/tag.py
index 80b4d5c3f1..23df204428 100644
--- a/paimon-python/pypaimon/tag/tag.py
+++ b/paimon-python/pypaimon/tag/tag.py
@@ -70,6 +70,7 @@ class Tag(Snapshot):
total_record_count=snapshot.total_record_count,
delta_record_count=snapshot.delta_record_count,
commit_user=snapshot.commit_user,
+ writer_version=snapshot.writer_version,
commit_identifier=snapshot.commit_identifier,
commit_kind=snapshot.commit_kind,
time_millis=snapshot.time_millis,
@@ -99,6 +100,7 @@ class Tag(Snapshot):
total_record_count=self.total_record_count,
delta_record_count=self.delta_record_count,
commit_user=self.commit_user,
+ writer_version=self.writer_version,
commit_identifier=self.commit_identifier,
commit_kind=self.commit_kind,
time_millis=self.time_millis,
diff --git a/paimon-python/pypaimon/tests/build_info_test.py
b/paimon-python/pypaimon/tests/build_info_test.py
new file mode 100644
index 0000000000..c1d7d1e385
--- /dev/null
+++ b/paimon-python/pypaimon/tests/build_info_test.py
@@ -0,0 +1,137 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from pypaimon import build_info
+
+
+class BuildInfoTest(unittest.TestCase):
+
+ def test_full_version(self):
+ self.assertRegex(
+ build_info.full_version(),
+ r"^python-2\.1\.dev-(UNKNOWN|[0-9a-f]{40})$",
+ )
+
+ def test_embedded_full_version(self):
+ with tempfile.NamedTemporaryFile(mode="w", delete=False) as
version_file:
+ version_file.write(
+ "python-2.1.dev-0123456789012345678901234567890123456789\n")
+ path = version_file.name
+ try:
+ with patch.object(build_info, "_FULL_VERSION_FILE", path):
+ self.assertEqual(
+ "python-2.1.dev-0123456789012345678901234567890123456789",
+ build_info._load_full_version(),
+ )
+ finally:
+ os.remove(path)
+
+ @unittest.skipIf(shutil.which("git") is None, "Git is not available")
+ def test_does_not_discover_outer_repository(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ outer = os.path.join(tmp, "outer")
+ source = os.path.join(outer, "source")
+ os.makedirs(source)
+ subprocess.check_call(
+ ["git", "init", "-q", outer], stdout=subprocess.DEVNULL)
+ subprocess.check_call(
+ ["git", "-C", outer, "config", "user.name", "test"])
+ subprocess.check_call(
+ ["git", "-C", outer, "config", "user.email",
"[email protected]"])
+ subprocess.check_call(
+ ["git", "-C", outer, "commit", "-q", "--allow-empty", "-m",
"outer"])
+
+ fake_module = os.path.join(source, "pypaimon", "build_info.py")
+ with patch.object(build_info, "__file__", fake_module):
+ self.assertEqual("UNKNOWN", build_info.git_commit_id())
+
+ @unittest.skipIf(shutil.which("git") is None, "Git is not available")
+ def test_sdist_provenance_survives_downstream_git_repository(self):
+ project_root = os.path.dirname(os.path.dirname(os.path.dirname(
+ os.path.abspath(__file__))))
+ with tempfile.TemporaryDirectory() as tmp:
+ source = os.path.join(tmp, "paimon-python")
+ shutil.copytree(
+ project_root,
+ source,
+ ignore=shutil.ignore_patterns(
+ ".pytest_cache", "*.egg-info", "__pycache__", "build",
"dist"),
+ )
+ upstream_commit = self._init_git_repository(source, "upstream")
+
+ sdist_dir = os.path.join(tmp, "sdist")
+ os.makedirs(sdist_dir)
+ subprocess.check_call(
+ [sys.executable, "setup.py", "-q", "sdist", "--dist-dir",
sdist_dir],
+ cwd=source,
+ )
+ archives = [
+ os.path.join(sdist_dir, name)
+ for name in os.listdir(sdist_dir)
+ if name.endswith(".tar.gz")
+ ]
+ self.assertEqual(1, len(archives))
+
+ extracted_root = os.path.join(tmp, "extracted")
+ os.makedirs(extracted_root)
+ with tarfile.open(archives[0], "r:gz") as archive:
+ top_level = archive.getnames()[0].split("/")[0]
+ archive.extractall(extracted_root)
+ extracted = os.path.join(extracted_root, top_level)
+ embedded_file = os.path.join(extracted, "pypaimon",
"_full_version")
+ with open(embedded_file, "r") as full_version_file:
+ embedded = full_version_file.read().strip()
+ self.assertEqual("python-2.1.dev-" + upstream_commit, embedded)
+
+ downstream_commit = self._init_git_repository(extracted,
"downstream")
+ self.assertNotEqual(upstream_commit, downstream_commit)
+ build_lib = os.path.join(tmp, "build")
+ subprocess.check_call(
+ [sys.executable, "setup.py", "-q", "build_py", "--build-lib",
build_lib],
+ cwd=extracted,
+ )
+ with open(
+ os.path.join(build_lib, "pypaimon", "_full_version"),
+ "r",
+ ) as full_version_file:
+ self.assertEqual(embedded, full_version_file.read().strip())
+
+ @staticmethod
+ def _init_git_repository(path, message):
+ subprocess.check_call(["git", "init", "-q", path])
+ subprocess.check_call(["git", "-C", path, "config", "user.name",
"test"])
+ subprocess.check_call(
+ ["git", "-C", path, "config", "user.email", "[email protected]"])
+ subprocess.check_call(["git", "-C", path, "add", "."])
+ subprocess.check_call(
+ ["git", "-C", path, "commit", "-q", "-m", message])
+ return subprocess.check_output(
+ ["git", "-C", path, "rev-parse", "HEAD"]
+ ).decode("utf-8").strip()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/paimon-python/pypaimon/tests/changelog_manager_test.py
b/paimon-python/pypaimon/tests/changelog_manager_test.py
index 3f726db7a4..b4e2daaedb 100644
--- a/paimon-python/pypaimon/tests/changelog_manager_test.py
+++ b/paimon-python/pypaimon/tests/changelog_manager_test.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+import dataclasses
import os
import shutil
import tempfile
@@ -22,6 +23,7 @@ import unittest
from pypaimon import CatalogFactory, Schema
from pypaimon.changelog import Changelog, ChangelogManager
+from pypaimon.snapshot.snapshot import Snapshot
import pyarrow as pa
@@ -119,15 +121,28 @@ class TestChangelogManager(unittest.TestCase):
def test_changelog_from_snapshot(self):
"""Test that Changelog can be created from a Snapshot."""
+ snapshot = Snapshot(
+ version=3,
+ id=1,
+ schema_id=2,
+ base_manifest_list="base",
+ delta_manifest_list="delta",
+ total_record_count=10,
+ delta_record_count=5,
+ commit_user="user",
+ commit_identifier=3,
+ commit_kind="APPEND",
+ time_millis=1000,
+ writer_version="python-2.1.dev-commit-id",
+ )
- snapshot_manager = self.table.snapshot_manager()
- snapshot = snapshot_manager.get_latest_snapshot()
+ changelog = Changelog.from_snapshot(snapshot)
- if snapshot:
- changelog = Changelog.from_snapshot(snapshot)
- self.assertEqual(changelog.id, snapshot.id)
- self.assertEqual(changelog.schema_id, snapshot.schema_id)
- self.assertEqual(changelog.time_millis, snapshot.time_millis)
+ for field in dataclasses.fields(Snapshot):
+ self.assertEqual(
+ getattr(snapshot, field.name), getattr(changelog, field.name),
+ "Changelog dropped/changed Snapshot field
'{}'".format(field.name),
+ )
if __name__ == '__main__':
diff --git a/paimon-python/pypaimon/tests/system/snapshots_table_test.py
b/paimon-python/pypaimon/tests/system/snapshots_table_test.py
index 4e3e637c07..aadb69dcaa 100644
--- a/paimon-python/pypaimon/tests/system/snapshots_table_test.py
+++ b/paimon-python/pypaimon/tests/system/snapshots_table_test.py
@@ -56,6 +56,7 @@ def _seed_two_snapshots(table):
commit_identifier=100,
commit_kind="APPEND",
time_millis=_T0,
+ writer_version="python-2.1.dev-commit-one",
watermark=7,
)
snapshot_two = Snapshot(
@@ -115,7 +116,7 @@ class SnapshotsTableTest(unittest.TestCase):
("changelog_manifest_list", True),
("total_record_count", True), ("delta_record_count", True),
("changelog_record_count", True), ("watermark", True),
- ("next_row_id", True),
+ ("next_row_id", True), ("writer_version", True),
]
self.assertEqual([n for n, _ in expected],
[f.name for f in row_type.fields])
@@ -153,6 +154,8 @@ class SnapshotsTableTest(unittest.TestCase):
arrow_table.column("changelog_record_count").to_pylist())
self.assertEqual([None, None],
arrow_table.column("next_row_id").to_pylist())
+ self.assertEqual(["python-2.1.dev-commit-one", None],
+ arrow_table.column("writer_version").to_pylist())
times = arrow_table.column("commit_time").to_pylist()
self.assertIsInstance(times[0], datetime.datetime)
diff --git a/paimon-python/pypaimon/tests/tag_ttl_serde_test.py
b/paimon-python/pypaimon/tests/tag_ttl_serde_test.py
index 23af669768..4bcbd60576 100644
--- a/paimon-python/pypaimon/tests/tag_ttl_serde_test.py
+++ b/paimon-python/pypaimon/tests/tag_ttl_serde_test.py
@@ -72,6 +72,7 @@ def _full_snapshot():
total_record_count=100,
delta_record_count=10,
commit_user="u",
+ writer_version="python-2.1.dev-commit-id",
commit_identifier=42,
commit_kind="APPEND",
time_millis=1000,
diff --git a/paimon-python/pypaimon/tests/write/table_write_test.py
b/paimon-python/pypaimon/tests/write/table_write_test.py
index e02a688cf9..4009b25790 100644
--- a/paimon-python/pypaimon/tests/write/table_write_test.py
+++ b/paimon-python/pypaimon/tests/write/table_write_test.py
@@ -28,6 +28,7 @@ from pypaimon import CatalogFactory, Schema
import pyarrow as pa
from parameterized import parameterized
+from pypaimon.build_info import full_version as build_full_version
from pypaimon.common.json_util import JSON
from pypaimon.common.options.core_options import CoreOptions
from pypaimon.manifest.manifest_list_manager import ManifestListManager
@@ -426,8 +427,11 @@ class TableWriteTest(unittest.TestCase):
self.assertEqual(self.expected, actual)
# snapshot
- snapshot_json: str =
JSON.to_json(table.snapshot_manager().get_latest_snapshot())
+ snapshot = table.snapshot_manager().get_latest_snapshot()
+ snapshot_json: str = JSON.to_json(snapshot)
self.assertEqual(True, snapshot_json.__contains__("baseManifestList"))
+ self.assertEqual(build_full_version(), snapshot.writer_version)
+ self.assertEqual(True, snapshot_json.__contains__("writerVersion"))
self.assertEqual(False, snapshot_json.__contains__("nextRowId"))
def test_write_row_append_only_partitioned_table(self):
diff --git a/paimon-python/pypaimon/write/file_store_commit.py
b/paimon-python/pypaimon/write/file_store_commit.py
index adcfa79148..2f64c59365 100644
--- a/paimon-python/pypaimon/write/file_store_commit.py
+++ b/paimon-python/pypaimon/write/file_store_commit.py
@@ -21,6 +21,7 @@ import time
import uuid
from typing import Dict, List, Optional
+from pypaimon.build_info import full_version as build_full_version
from pypaimon.common.options.core_options import CoreOptions
from pypaimon.common.predicate_builder import PredicateBuilder
from pypaimon.manifest.manifest_file_manager import ManifestFileManager
@@ -761,6 +762,7 @@ class FileStoreCommit:
total_record_count=total_record_count,
delta_record_count=delta_record_count,
commit_user=self.commit_user,
+ writer_version=build_full_version(),
commit_identifier=commit_identifier,
commit_kind=commit_kind,
time_millis=int(time.time() * 1000),
diff --git a/paimon-python/setup.py b/paimon-python/setup.py
index 27e5efe6e2..8907ef4d88 100644
--- a/paimon-python/setup.py
+++ b/paimon-python/setup.py
@@ -23,10 +23,75 @@ import sys
import tarfile
import tempfile
from setuptools import find_packages, setup
+from setuptools.command.build_py import build_py
+from setuptools.command.sdist import sdist
+
+PYTHON_ROOT = os.path.dirname(os.path.abspath(__file__))
+FULL_VERSION_FILE = os.path.join(PYTHON_ROOT, "pypaimon", "_full_version")
+UNKNOWN_COMMIT_ID = "UNKNOWN"
VERSION = "2.1.dev"
+def _repository_root():
+ parent = os.path.dirname(PYTHON_ROOT)
+ if os.path.basename(PYTHON_ROOT) == "paimon-python" and os.path.exists(
+ os.path.join(parent, "pom.xml")):
+ return parent
+ return PYTHON_ROOT
+
+
+def _git_output(args):
+ repository_root = _repository_root()
+ env = os.environ.copy()
+ env["GIT_CEILING_DIRECTORIES"] = os.path.dirname(repository_root)
+ try:
+ return subprocess.check_output(
+ ["git", "-C", repository_root] + args,
+ stderr=subprocess.DEVNULL,
+ env=env,
+ ).decode("utf-8").strip()
+ except Exception:
+ return None
+
+
+def _full_version():
+ try:
+ with open(FULL_VERSION_FILE, "r") as full_version_file:
+ embedded = full_version_file.read().strip()
+ if embedded:
+ return embedded
+ except OSError:
+ pass
+
+ git_commit_id = _git_output(["rev-parse", "HEAD"])
+ if git_commit_id is None:
+ git_commit_id = UNKNOWN_COMMIT_ID
+ return "python-{}-{}".format(VERSION, git_commit_id)
+
+
+def _write_full_version(root):
+ package_dir = os.path.join(root, "pypaimon")
+ if not os.path.exists(package_dir):
+ os.makedirs(package_dir)
+ with open(os.path.join(package_dir, "_full_version"), "w") as
full_version_file:
+ full_version_file.write(_full_version() + "\n")
+
+
+class PaimonBuildPy(build_py):
+
+ def run(self):
+ build_py.run(self)
+ _write_full_version(self.build_lib)
+
+
+class PaimonSdist(sdist):
+
+ def make_release_tree(self, base_dir, files):
+ sdist.make_release_tree(self, base_dir, files)
+ _write_full_version(base_dir)
+
+
def get_dev_version():
"""Generate dev version with commit date.
Format: 2.1.devYYYYMMDD (e.g. 2.1.dev20260415)
@@ -37,10 +102,10 @@ def get_dev_version():
return None
try:
- date_str = subprocess.check_output(
- ["git", "log", "-1", "--format=%cd", "--date=format:%Y%m%d"],
- stderr=subprocess.DEVNULL
- ).decode("utf-8").strip()
+ date_str = _git_output(
+ ["log", "-1", "--format=%cd", "--date=format:%Y%m%d"])
+ if date_str is None:
+ raise RuntimeError("Git commit date is unavailable")
except Exception:
print("Warning: git not available, skipping dev package.")
return None
@@ -104,6 +169,18 @@ def _build_dev_package():
with open(setup_py, "w") as f:
f.write(content)
+ # Keep the embedded full version consistent with the dev package
version.
+ full_version_file = os.path.join(dev_dir, "pypaimon", "_full_version")
+ if os.path.exists(full_version_file):
+ with open(full_version_file, "r") as f:
+ content = f.read()
+ version_prefix = "python-{}-".format(VERSION)
+ if content.startswith(version_prefix):
+ content = "python-{}-{}".format(
+ dev_version, content[len(version_prefix):])
+ with open(full_version_file, "w") as f:
+ f.write(content)
+
dev_tar = os.path.join("dist", dev_name + ".tar.gz")
with tarfile.open(dev_tar, "w:gz") as tar:
tar.add(dev_dir, arcname=dev_name)
@@ -147,6 +224,8 @@ setup(
version=VERSION,
packages=PACKAGES,
include_package_data=True,
+ package_data={"pypaimon": ["_full_version"]},
+ cmdclass={"build_py": PaimonBuildPy, "sdist": PaimonSdist},
install_requires=install_requires,
entry_points={
'console_scripts': [
diff --git
a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorReadTest.java
b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorReadTest.java
index dd13cd365b..2bd1833705 100644
---
a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorReadTest.java
+++
b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorReadTest.java
@@ -124,6 +124,7 @@ public class SparkDataEvolutionVectorReadTest {
null,
null,
"user",
+ null,
0L,
Snapshot.CommitKind.APPEND,
0L,
