vinishjail97 commented on code in PR #768:
URL: https://github.com/apache/incubator-xtable/pull/768#discussion_r3112400816
##########
xtable-core/src/main/java/org/apache/xtable/parquet/ParquetConversionSource.java:
##########
@@ -162,14 +167,17 @@ public TableChange getTableChangeForCommit(Long
modificationTime) {
parquetFiles
.filter(fileStatus -> fileStatus.getModificationTime() >
modificationTime)
.collect(Collectors.toList());
- InternalTable internalTable = getMostRecentTable(parquetFiles);
+ InternalTable internalTable =
getMostRecentTable(getParquetFiles(hadoopConf, basePath));
Review Comment:
Boundary inconsistency nearby (line 168 filter): this method uses
`fileStatus.getModificationTime() > modificationTime`, but
`ParquetDataManager.getParquetFilesMetadataAfterTime(long)` filters with `>=`.
A file whose mod time equals the prior sync instant is silently dropped from
the diff here. Pick one semantic (ideally `>` everywhere — the instant is
exclusive) and apply consistently across both classes?
##########
xtable-core/src/test/java/org/apache/xtable/parquet/TestParquetDataManager.java:
##########
@@ -0,0 +1,453 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.parquet;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class TestParquetDataManager {
+
+ private static final Configuration CONF = new Configuration();
+
+ // Helper method to create a mock LocatedFileStatus
+ private static LocatedFileStatus createMockFileStatus(String name, long
modTime) {
+ LocatedFileStatus mockStatus = mock(LocatedFileStatus.class);
+ org.apache.hadoop.fs.Path mockPath = mock(org.apache.hadoop.fs.Path.class);
+ when(mockPath.getName()).thenReturn(name);
+ when(mockStatus.getPath()).thenReturn(mockPath);
+ when(mockStatus.getModificationTime()).thenReturn(modTime);
+ when(mockStatus.getLen()).thenReturn(1024L);
+ return mockStatus;
+ }
+
+ private static class StubbedRemoteIterator implements
RemoteIterator<LocatedFileStatus> {
+ private final Iterator<LocatedFileStatus> files;
+
+ public StubbedRemoteIterator(List<LocatedFileStatus> files) {
+ this.files = files.iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return files.hasNext();
+ }
+
+ @Override
+ public LocatedFileStatus next() {
+ return files.next();
+ }
+ }
+
+ // Helper method to create a mock FileSystem
+ private static FileSystem
createMockFileSystem(RemoteIterator<LocatedFileStatus> iterator)
+ throws IOException {
+ FileSystem mockFs = mock(FileSystem.class);
+ when(mockFs.listFiles(any(org.apache.hadoop.fs.Path.class),
anyBoolean())).thenReturn(iterator);
+ return mockFs;
+ }
+
+ @Test
+ void testGetMostRecentParquetFile_withMultipleFiles() throws IOException {
+ LocatedFileStatus file1 = createMockFileStatus("file1.parquet", 1000L);
+ LocatedFileStatus file2 = createMockFileStatus("file2.parquet", 3000L);
+ LocatedFileStatus file3 = createMockFileStatus("file3.parquet", 2000L);
+
+ RemoteIterator<LocatedFileStatus> iterator =
+ new StubbedRemoteIterator(Arrays.asList(file1, file2, file3));
+ FileSystem mockFs = createMockFileSystem(iterator);
+
+ ParquetDataManager manager = new ParquetDataManager(CONF, "test-path",
mockFs);
+
+ ParquetFileInfo result = manager.getMostRecentParquetFile();
+
+ assertNotNull(result);
+ assertEquals(3000L, result.getModificationTime());
+ }
+
+ @Test
+ void testGetMostRecentParquetFile_noFiles() throws IOException {
+ RemoteIterator<LocatedFileStatus> iterator = new StubbedRemoteIterator(new
ArrayList<>());
+ FileSystem mockFs = createMockFileSystem(iterator);
+
+ ParquetDataManager manager = new ParquetDataManager(CONF, "test-path",
mockFs);
+
+ IllegalStateException exception =
+ assertThrows(IllegalStateException.class,
manager::getMostRecentParquetFile);
+ assertEquals("No files found", exception.getMessage());
+ }
+
+ @Test
+ void testGetParquetDataFileAt_exactMatch() throws IOException {
+ LocatedFileStatus file1 = createMockFileStatus("file1.parquet", 1000L);
+ LocatedFileStatus file2 = createMockFileStatus("file2.parquet", 2000L);
+ LocatedFileStatus file3 = createMockFileStatus("file3.parquet", 3000L);
+
+ RemoteIterator<LocatedFileStatus> iterator =
+ new StubbedRemoteIterator(Arrays.asList(file1, file2, file3));
+ FileSystem mockFs = createMockFileSystem(iterator);
+
+ ParquetDataManager manager = new ParquetDataManager(CONF, "test-path",
mockFs);
+
+ ParquetFileInfo result = manager.getParquetDataFileAt(2000L);
+
+ assertNotNull(result);
+ assertEquals(2000L, result.getModificationTime());
+ }
+
+ @Test
+ void testGetParquetDataFileAt_firstAfterTime() throws IOException {
+ LocatedFileStatus file1 = createMockFileStatus("file1.parquet", 1000L);
+ LocatedFileStatus file2 = createMockFileStatus("file2.parquet", 2500L);
+ LocatedFileStatus file3 = createMockFileStatus("file3.parquet", 3000L);
+
+ RemoteIterator<LocatedFileStatus> iterator =
+ new StubbedRemoteIterator(Arrays.asList(file1, file2, file3));
+ FileSystem mockFs = createMockFileSystem(iterator);
+
+ ParquetDataManager manager = new ParquetDataManager(CONF, "test-path",
mockFs);
+
+ ParquetFileInfo result = manager.getParquetDataFileAt(2000L);
+
+ assertNotNull(result);
+ assertEquals(2500L, result.getModificationTime());
+ }
+
+ @Test
+ void testGetParquetDataFileAt_multipleAfterTime() throws IOException {
+ LocatedFileStatus file1 = createMockFileStatus("file1.parquet", 1000L);
+ LocatedFileStatus file2 = createMockFileStatus("file2.parquet", 2500L);
+ LocatedFileStatus file3 = createMockFileStatus("file3.parquet", 3000L);
+
+ RemoteIterator<LocatedFileStatus> iterator =
+ new StubbedRemoteIterator(Arrays.asList(file1, file2, file3));
+ FileSystem mockFs = createMockFileSystem(iterator);
+
+ ParquetDataManager manager = new ParquetDataManager(CONF, "test-path",
mockFs);
+
+ ParquetFileInfo result = manager.getParquetDataFileAt(2000L);
+
+ assertNotNull(result);
+ assertEquals(2500L, result.getModificationTime());
+ }
+
+ @Test
+ void testGetParquetDataFileAt_noMatch() throws IOException {
+ RemoteIterator<LocatedFileStatus> iterator = new
StubbedRemoteIterator(Collections.emptyList());
+ FileSystem mockFs = createMockFileSystem(iterator);
+
+ ParquetDataManager manager = new ParquetDataManager(CONF, "test-path",
mockFs);
+
+ IllegalStateException exception =
+ assertThrows(IllegalStateException.class, () ->
manager.getParquetDataFileAt(5000L));
+ assertTrue(exception.getMessage().contains("No file found at or after
5000"));
+ }
+
+ @Test
+ void testGetParquetDataFileAt_allBefore() throws IOException {
+ LocatedFileStatus file1 = createMockFileStatus("file1.parquet", 1000L);
+ LocatedFileStatus file2 = createMockFileStatus("file2.parquet", 2000L);
+
+ RemoteIterator<LocatedFileStatus> iterator =
+ new StubbedRemoteIterator(Arrays.asList(file1, file2));
+ FileSystem mockFs = createMockFileSystem(iterator);
+
+ ParquetDataManager manager = new ParquetDataManager(CONF, "test-path",
mockFs);
+
+ IllegalStateException exception =
+ assertThrows(IllegalStateException.class, () ->
manager.getParquetDataFileAt(3000L));
+ assertTrue(exception.getMessage().contains("No file found at or after
3000"));
+ }
+
+ @Test
+ void testGetCurrentFilesInfo_multipleFiles() throws IOException {
+ LocatedFileStatus file1 = createMockFileStatus("file1.parquet", 1000L);
+ LocatedFileStatus file2 = createMockFileStatus("file2.parquet", 2000L);
+ LocatedFileStatus file3 = createMockFileStatus("file3.parquet", 3000L);
+
+ RemoteIterator<LocatedFileStatus> iterator =
+ new StubbedRemoteIterator(Arrays.asList(file1, file2, file3));
+ FileSystem mockFs = createMockFileSystem(iterator);
+
+ ParquetDataManager manager = new ParquetDataManager(CONF, "test-path",
mockFs);
+
+ Stream<ParquetFileInfo> result = manager.getCurrentFilesInfo();
+
+ assertNotNull(result);
+ List<ParquetFileInfo> fileList = result.collect(Collectors.toList());
+ assertEquals(3, fileList.size());
+ assertEquals(1000L, fileList.get(0).getModificationTime());
Review Comment:
Nit — asserting positional ordering here relies on `RemoteIterator` +
`Collectors.toList()` preserving the mock insertion order. In production, FS
listing order is platform-dependent and not guaranteed. Either sort inside
`getCurrentFilesInfo()` (and document the ordering contract) or switch these
assertions to `Set<Long>` comparisons. Same pattern applies to
`testGetParquetFilesMetadataAfterTime_someMatch` and `_exactTimeMatch`.
##########
xtable-core/src/test/java/org/apache/xtable/parquet/ITParquetConversionSource.java:
##########
@@ -418,6 +432,124 @@ private void checkDatasetEquivalence(
}
}
+ @Test
+ void testIncrementalSyncWithMultiplePartitions() throws IOException {
+
+ Configuration conf = sparkSession.sparkContext().hadoopConfiguration();
+
+ StructType schema =
+ DataTypes.createStructType(
+ new StructField[] {
+ DataTypes.createStructField("id", DataTypes.IntegerType, false),
+ DataTypes.createStructField("value", DataTypes.StringType,
false),
+ DataTypes.createStructField("year", DataTypes.IntegerType,
false),
+ DataTypes.createStructField("month", DataTypes.IntegerType,
false)
+ });
+ List<Row> data =
+ Arrays.asList(
+ RowFactory.create(100, "A", 2026, 12),
+ RowFactory.create(101, "AA", 2026, 12),
+ RowFactory.create(102, "CB", 2027, 11),
+ RowFactory.create(103, "BA", 2027, 11));
+
+ Dataset<Row> dfInit = sparkSession.createDataFrame(data, schema);
+ Path fixedPath = Paths.get("target", "fixed-parquet-data",
"parquet_table_test_2");
+ // String outputPath = fixedPath.toString();
+ Dataset<Row> df = dfInit.withColumn("full_date", expr("make_date(year,
month, 1)"));
+ String outputPath =
+ new
java.io.File("target/fixed-parquet-data/parquet_table_test_2").getAbsolutePath();
+ df.coalesce(1).write().partitionBy("year",
"month").mode("overwrite").parquet(outputPath);
+
+ // test find files to sync
+ FileSystem fs = FileSystem.get(fixedPath.toUri(), conf);
+ // set the modification time to the table file
+ // update modificationTime for file to append
+ // many partitions case
+ List<String> newPartitions = Arrays.asList("year=2026/month=12",
"year=2027/month=11");
+ long targetModificationTime = System.currentTimeMillis() - 360000;
+ long newModificationTime = System.currentTimeMillis() - 50000;
+ long testTime = System.currentTimeMillis() - 90000; // between two prev
times
+ for (String partition : newPartitions) {
+ org.apache.hadoop.fs.Path partitionPath =
+ new org.apache.hadoop.fs.Path(outputPath, partition);
+ if (fs.exists(partitionPath)) {
+ updateModificationTimeRecursive(fs, partitionPath,
targetModificationTime);
+ }
+ }
+ // create new file to append using Spark
+ List<Row> futureDataToSync =
+ Arrays.asList(
+ RowFactory.create(101, "A", 2026, 12),
+ RowFactory.create(301, "D", 2027, 11),
+ RowFactory.create(302, "DA", 2027, 11));
+ Dataset<Row> dfToSyncInit = sparkSession.createDataFrame(futureDataToSync,
schema);
+ Dataset<Row> dfToSync = dfToSyncInit.withColumn("full_date",
expr("make_date(year, month, 1)"));
+ dfToSync.coalesce(1).write().partitionBy("year",
"month").mode("append").parquet(outputPath);
+
+ // conversionSource operations
+ Properties sourceProperties = new Properties();
+ String partitionConfig = "full_date:MONTH:year=yyyy/month=MM";
+ sourceProperties.put(PARTITION_FIELD_SPEC_CONFIG, partitionConfig);
+
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name("parquet_table_test_2")
+ .basePath(fixedPath.toAbsolutePath().toString())
+ .additionalProperties(sourceProperties)
+ .formatName(TableFormat.PARQUET)
+ .build();
+
+ ParquetConversionSourceProvider conversionSourceProvider =
+ new ParquetConversionSourceProvider();
+ conversionSourceProvider.init(conf);
+ ParquetConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+
+ for (String partition : newPartitions) {
+ org.apache.hadoop.fs.Path partitionPath =
+ new org.apache.hadoop.fs.Path(outputPath, partition);
+
+ RemoteIterator<LocatedFileStatus> it = fs.listFiles(partitionPath,
false);
+ while (it.hasNext()) {
+ LocatedFileStatus fileStatus = it.next();
+
+ if (fileStatus.getModificationTime() > newModificationTime) {
+ fs.setTimes(fileStatus.getPath(), newModificationTime, -1);
+ } else {
+ fs.setTimes(fileStatus.getPath(), targetModificationTime, -1);
+ }
+ }
+ fs.setTimes(partitionPath, newModificationTime, -1);
+ }
+
+ InternalTable result = conversionSource.getTable(newModificationTime);
+ assertEquals(
+ Instant.ofEpochMilli(newModificationTime).toString(),
+ result.getLatestCommitTime().toString());
+ assertEquals("parquet_table_test_2", result.getName());
+ assertEquals(TableFormat.PARQUET, result.getTableFormat());
+ assertNotNull(result.getReadSchema());
+ InternalSnapshot snapshot = conversionSource.getCurrentSnapshot();
+ assertNotNull(snapshot);
+ TableChange changes =
conversionSource.getTableChangeForCommit(newModificationTime);
+ assertNotNull(changes);
+ Instant instantBeforeFirstSnapshot =
+
Instant.ofEpochMilli(snapshot.getTable().getLatestCommitTime().toEpochMilli());
+ assertEquals(instantBeforeFirstSnapshot.toEpochMilli(),
newModificationTime);
+
assertTrue(conversionSource.isIncrementalSyncSafeFrom(Instant.ofEpochMilli(testTime)));
Review Comment:
This test asserts `changes` is non-null and
`isIncrementalSyncSafeFrom(testTime) == true`, but does not validate the actual
`filesAdded` set?
Please build the expected set of physical paths (or `InternalDataFile`s) and
`assertEquals` against `changes.getFilesDiff().getFilesAdded()`. Also add a
negative case: `isIncrementalSyncSafeFrom` returns `false` when earliest file
is newer than the instant, and when the table is empty. Without these, the test
would pass even if the diff were silently empty.
##########
xtable-core/src/test/java/org/apache/xtable/parquet/ITParquetConversionSource.java:
##########
@@ -418,6 +432,124 @@ private void checkDatasetEquivalence(
}
}
+ @Test
+ void testIncrementalSyncWithMultiplePartitions() throws IOException {
+
+ Configuration conf = sparkSession.sparkContext().hadoopConfiguration();
+
+ StructType schema =
+ DataTypes.createStructType(
+ new StructField[] {
+ DataTypes.createStructField("id", DataTypes.IntegerType, false),
+ DataTypes.createStructField("value", DataTypes.StringType,
false),
+ DataTypes.createStructField("year", DataTypes.IntegerType,
false),
+ DataTypes.createStructField("month", DataTypes.IntegerType,
false)
+ });
+ List<Row> data =
+ Arrays.asList(
+ RowFactory.create(100, "A", 2026, 12),
+ RowFactory.create(101, "AA", 2026, 12),
+ RowFactory.create(102, "CB", 2027, 11),
+ RowFactory.create(103, "BA", 2027, 11));
+
+ Dataset<Row> dfInit = sparkSession.createDataFrame(data, schema);
+ Path fixedPath = Paths.get("target", "fixed-parquet-data",
"parquet_table_test_2");
Review Comment:
Relative path `Paths.get("target", "fixed-parquet-data",
"parquet_table_test_2")` pollutes the workspace across test runs, isn't cleaned
up, and makes the test order-dependent when re-run without `./gradlew clean`.
This class already uses the `@TempDir` pattern — please use it here too. Also
drop the commented-out `// String outputPath = fixedPath.toString();` on line
457.
##########
xtable-core/src/main/java/org/apache/xtable/parquet/ParquetDataManager.java:
##########
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.parquet;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Paths;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.util.functional.RemoteIterators;
+
+import org.apache.xtable.exception.ReadException;
+
+/**
+ * Manages Parquet File's Metadata
+ *
+ * <p>This class provides functions to handle Parquet metadata, creating
metadata objects from
+ * parquet files and filtering the files based on the modification times.
+ */
+@Log4j2
+@RequiredArgsConstructor
+public class ParquetDataManager {
+ private final Configuration hadoopConf;
+ private final String basePath;
+ private final FileSystem fileSystem;
+
+ public ParquetDataManager(Configuration hadoopConf, String basePath) {
+ this.hadoopConf = hadoopConf;
+ this.basePath = basePath;
+ try {
+ URI uri = Paths.get(basePath).toUri();
Review Comment:
Blocking — `Paths.get(basePath).toUri()` uses the JDK's *default*
FileSystemProvider (local filesystem). For `s3://bucket/...`, `gs://...`,
`abfs://...`, this either throws `InvalidPathException` (colon is invalid on
some platforms) or silently produces a `file://` URI, after which
`FileSystem.get(uri, hadoopConf)` returns the local Hadoop FS — not
S3A/GCS/ABFS. That breaks all cloud storage users.
Use Hadoop's `Path` instead: `URI uri = new Path(basePath).toUri();`. This
mirrors how Hudi and the rest of xtable resolve base paths for cloud stores.
##########
xtable-core/src/main/java/org/apache/xtable/parquet/ParquetDataManager.java:
##########
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.parquet;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Paths;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.util.functional.RemoteIterators;
+
+import org.apache.xtable.exception.ReadException;
+
+/**
+ * Manages Parquet File's Metadata
+ *
+ * <p>This class provides functions to handle Parquet metadata, creating
metadata objects from
+ * parquet files and filtering the files based on the modification times.
+ */
+@Log4j2
+@RequiredArgsConstructor
Review Comment:
Nit — exposing both the Lombok-generated 3-arg ctor
(`@RequiredArgsConstructor`) and an explicit 2-arg ctor creates ambiguity about
the public API. Production code calls the 2-arg form; tests call the 3-arg
form. Consider dropping `@RequiredArgsConstructor` and either annotating the
3-arg ctor `@VisibleForTesting` or using a package-private static factory
method for tests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]