This is an automated email from the ASF dual-hosted git repository.
vinishjail97 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
The following commit(s) were added to refs/heads/main by this push:
new b73a1185 fix: Keep absolute paths for data files outside the table
base path in Delta targets (#957)
b73a1185 is described below
commit b73a118522fd5b3496a3859bbd6aeccdee2e8064
Author: Vinish Reddy <[email protected]>
AuthorDate: Wed Oct 7 06:15:46 2026 +0530
fix: Keep absolute paths for data files outside the table base path in
Delta targets (#957)
* fix: Keep absolute paths for data files outside the table base path in
Delta targets
Both Delta targets passed every data file to PathUtils.getRelativePath,
which assumes the file is under the table base path. For a file outside
the base path it removed characters by length, so the Delta log got a
broken path such as nch.db/spans/data/file.parquet while the sync still
reported success. This happens when the target base path differs from
the location of the source data files, for example an Iceberg table
with write.data.path set, or with files added by add_files or snapshot.
Add DeltaPathUtils.getPathForDeltaLog, the write-side counterpart of
getFullPathToFile. It records a file under the base path relative to
it, comparing the paths without the scheme to handle s3 against s3a,
and records any other file with its absolute path.
* Skip Path parsing for Delta paths without a scheme
---
.../delta/DeltaDataFileUpdatesExtractor.java | 5 +--
.../org/apache/xtable/delta/DeltaPathUtils.java | 37 ++++++++++++++++++++--
.../DeltaKernelDataFileUpdatesExtractor.java | 6 ++--
.../apache/xtable/delta/TestDeltaPathUtils.java | 32 +++++++++++++++++++
4 files changed, 70 insertions(+), 10 deletions(-)
diff --git
a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaDataFileUpdatesExtractor.java
b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaDataFileUpdatesExtractor.java
index 71cebf9a..a338ec34 100644
---
a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaDataFileUpdatesExtractor.java
+++
b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaDataFileUpdatesExtractor.java
@@ -47,7 +47,6 @@ import org.apache.xtable.model.storage.InternalDataFile;
import org.apache.xtable.model.storage.InternalFile;
import org.apache.xtable.model.storage.InternalFilesDiff;
import org.apache.xtable.model.storage.PartitionFileGroup;
-import org.apache.xtable.paths.PathUtils;
@Builder
public class DeltaDataFileUpdatesExtractor {
@@ -116,9 +115,7 @@ public class DeltaDataFileUpdatesExtractor {
InternalDataFile dataFile, InternalSchema schema, String tableBasePath) {
return Stream.of(
new AddFile(
- // Delta Lake supports relative and absolute paths in theory but
relative paths seem
- // more commonly supported by query engines in our testing
- PathUtils.getRelativePath(dataFile.getPhysicalPath(),
tableBasePath),
+ DeltaPathUtils.getPathForDeltaLog(tableBasePath,
dataFile.getPhysicalPath()),
convertJavaMapToScala(deltaPartitionExtractor.partitionValueSerialization(dataFile)),
dataFile.getFileSizeBytes(),
dataFile.getLastModified(),
diff --git
a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaPathUtils.java
b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaPathUtils.java
index 141aac33..f690c379 100644
--- a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaPathUtils.java
+++ b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaPathUtils.java
@@ -18,11 +18,16 @@
package org.apache.xtable.delta;
+import java.net.URI;
+import java.util.Objects;
+
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import org.apache.hadoop.fs.Path;
+import org.apache.xtable.paths.PathUtils;
+
/**
* Shared path resolution for Delta log actions. Used by both Delta Standalone
and Delta Kernel
* implementations.
@@ -51,8 +56,36 @@ public class DeltaPathUtils {
: tableBasePath + Path.SEPARATOR + dataFilePath;
}
+ /**
+ * Returns the path to record in an AddFile or RemoveFile action for a data
file.
+ *
+ * <p>A file under the table base path is recorded relative to it, since
relative paths seem more
+ * commonly supported by query engines. A file outside the base path, for
example one written to a
+ * separate data location of an Iceberg source, is recorded with its
absolute path, since a
+ * relative path cannot point to it.
+ *
+ * @param tableBasePath the table base path
+ * @param dataFilePath the path to the data file, relative or absolute
+ * @return the path to record in the Delta log
+ */
+ public static String getPathForDeltaLog(String tableBasePath, String
dataFilePath) {
+ return isUnderBasePath(tableBasePath, dataFilePath)
+ ? PathUtils.getRelativePath(dataFilePath, tableBasePath)
+ : dataFilePath;
+ }
+
+ private static boolean isUnderBasePath(String tableBasePath, String
dataFilePath) {
+ if (!dataFilePath.contains(":")) {
+ return true;
+ }
+ URI file = new Path(dataFilePath).toUri();
+ // compare without the scheme to handle differences like s3 vs s3a
+ URI base = new Path(tableBasePath).toUri();
+ return Objects.equals(base.getAuthority(), file.getAuthority())
+ && file.getPath().startsWith(base.getPath() + Path.SEPARATOR);
+ }
+
private static boolean isAbsolutePath(String dataFilePath) {
- return dataFilePath.startsWith(Path.SEPARATOR)
- || new Path(dataFilePath).toUri().getScheme() != null;
+ return dataFilePath.startsWith(Path.SEPARATOR) ||
dataFilePath.contains(":");
}
}
diff --git
a/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileUpdatesExtractor.java
b/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileUpdatesExtractor.java
index e9412128..f1bf9f63 100644
---
a/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileUpdatesExtractor.java
+++
b/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileUpdatesExtractor.java
@@ -46,6 +46,7 @@ import io.delta.kernel.statistics.DataFileStatistics;
import io.delta.kernel.types.StructType;
import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.delta.DeltaPathUtils;
import org.apache.xtable.exception.ReadException;
import org.apache.xtable.model.schema.InternalField;
import org.apache.xtable.model.schema.InternalSchema;
@@ -57,7 +58,6 @@ import org.apache.xtable.model.storage.InternalDataFile;
import org.apache.xtable.model.storage.InternalFile;
import org.apache.xtable.model.storage.InternalFilesDiff;
import org.apache.xtable.model.storage.PartitionFileGroup;
-import org.apache.xtable.paths.PathUtils;
import org.apache.xtable.spi.extractor.DataFileIterator;
@Builder
@@ -205,9 +205,7 @@ public class DeltaKernelDataFileUpdatesExtractor {
Row addFileRow =
AddFile.createAddFileRow(
physicalSchema,
- // Delta Lake supports relative and absolute paths in theory but
relative paths seem
- // more commonly supported by query engines in our testing
- PathUtils.getRelativePath(dataFile.getPhysicalPath(),
tableBasePath),
+ DeltaPathUtils.getPathForDeltaLog(tableBasePath,
dataFile.getPhysicalPath()),
partitionValues,
dataFile.getFileSizeBytes(),
dataFile.getLastModified(),
diff --git
a/xtable-core/src/test/java/org/apache/xtable/delta/TestDeltaPathUtils.java
b/xtable-core/src/test/java/org/apache/xtable/delta/TestDeltaPathUtils.java
index 3defd993..b1223f98 100644
--- a/xtable-core/src/test/java/org/apache/xtable/delta/TestDeltaPathUtils.java
+++ b/xtable-core/src/test/java/org/apache/xtable/delta/TestDeltaPathUtils.java
@@ -58,4 +58,36 @@ class TestDeltaPathUtils {
void getFullPathToFile(String tableBasePath, String dataFilePath, String
expected) {
assertEquals(expected, DeltaPathUtils.getFullPathToFile(tableBasePath,
dataFilePath));
}
+
+ private static Stream<Arguments> deltaLogPathCases() {
+ return Stream.of(
+ // files under the base path are recorded relative to it
+ Arguments.of("s3a://bucket/tab", "s3a://bucket/tab/part-0.parquet",
"part-0.parquet"),
+ Arguments.of(
+ "s3a://bucket/tab",
+ "s3://bucket/tab/year=2024/part-0.parquet",
+ "year=2024/part-0.parquet"),
+ Arguments.of("s3://bucket/tab", "s3a://bucket/tab/part-0.parquet",
"part-0.parquet"),
+ Arguments.of("file:///tmp/tab", "file:/tmp/tab/part-0.parquet",
"part-0.parquet"),
+ Arguments.of("s3a://bucket/tab", "part-0.parquet", "part-0.parquet"),
+ // files outside the base path keep their absolute path
+ Arguments.of(
+ "s3a://bucket/target/tab",
+ "s3://bucket/source/tab/data/part-0.parquet",
+ "s3://bucket/source/tab/data/part-0.parquet"),
+ Arguments.of(
+ "s3a://bucket/tab",
+ "s3a://other-bucket/tab/part-0.parquet",
+ "s3a://other-bucket/tab/part-0.parquet"),
+ Arguments.of(
+ "s3a://bucket/tab",
+ "s3a://bucket/tab2/part-0.parquet",
+ "s3a://bucket/tab2/part-0.parquet"));
+ }
+
+ @ParameterizedTest
+ @MethodSource("deltaLogPathCases")
+ void getPathForDeltaLog(String tableBasePath, String dataFilePath, String
expected) {
+ assertEquals(expected, DeltaPathUtils.getPathForDeltaLog(tableBasePath,
dataFilePath));
+ }
}
