the-other-tim-brown commented on code in PR #768: URL: https://github.com/apache/incubator-xtable/pull/768#discussion_r3253829981
########## xtable-core/src/test/java/org/apache/xtable/parquet/TestParquetDataManager.java: ########## @@ -0,0 +1,453 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.xtable.parquet; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.RemoteIterator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestParquetDataManager { + + private static final Configuration CONF = new Configuration(); + + // Helper method to create a mock LocatedFileStatus + private static LocatedFileStatus createMockFileStatus(String name, long modTime) { + LocatedFileStatus mockStatus = mock(LocatedFileStatus.class); + org.apache.hadoop.fs.Path mockPath = mock(org.apache.hadoop.fs.Path.class); + when(mockPath.getName()).thenReturn(name); + when(mockStatus.getPath()).thenReturn(mockPath); + when(mockStatus.getModificationTime()).thenReturn(modTime); + when(mockStatus.getLen()).thenReturn(1024L); + return mockStatus; + } + + private static class StubbedRemoteIterator implements RemoteIterator<LocatedFileStatus> { + private final Iterator<LocatedFileStatus> files; + + public StubbedRemoteIterator(List<LocatedFileStatus> files) { + this.files = files.iterator(); + } + + @Override + public boolean hasNext() { + return files.hasNext(); + } + + @Override + public LocatedFileStatus next() { + return files.next(); + } + } + + // Helper method to create a mock FileSystem + private static FileSystem createMockFileSystem(RemoteIterator<LocatedFileStatus> iterator) + throws IOException { + FileSystem mockFs = mock(FileSystem.class); + when(mockFs.listFiles(any(org.apache.hadoop.fs.Path.class), anyBoolean())).thenReturn(iterator); + return mockFs; + } + + @Test + void testGetMostRecentParquetFile_withMultipleFiles() throws IOException { + LocatedFileStatus file1 = createMockFileStatus("file1.parquet", 1000L); + LocatedFileStatus file2 = createMockFileStatus("file2.parquet", 3000L); + LocatedFileStatus file3 = createMockFileStatus("file3.parquet", 2000L); + + RemoteIterator<LocatedFileStatus> iterator = + new StubbedRemoteIterator(Arrays.asList(file1, file2, file3)); + FileSystem mockFs = createMockFileSystem(iterator); + + ParquetDataManager manager = new ParquetDataManager(CONF, "test-path", mockFs); + + ParquetFileInfo result = manager.getMostRecentParquetFile(); + + assertNotNull(result); + assertEquals(3000L, result.getModificationTime()); + } + + @Test + void testGetMostRecentParquetFile_noFiles() throws IOException { + RemoteIterator<LocatedFileStatus> iterator = new StubbedRemoteIterator(new ArrayList<>()); + FileSystem mockFs = createMockFileSystem(iterator); + + ParquetDataManager manager = new ParquetDataManager(CONF, "test-path", mockFs); + + IllegalStateException exception = + assertThrows(IllegalStateException.class, manager::getMostRecentParquetFile); + assertEquals("No files found", exception.getMessage()); + } + + @Test + void testGetParquetDataFileAt_exactMatch() throws IOException { + LocatedFileStatus file1 = createMockFileStatus("file1.parquet", 1000L); + LocatedFileStatus file2 = createMockFileStatus("file2.parquet", 2000L); + LocatedFileStatus file3 = createMockFileStatus("file3.parquet", 3000L); + + RemoteIterator<LocatedFileStatus> iterator = + new StubbedRemoteIterator(Arrays.asList(file1, file2, file3)); + FileSystem mockFs = createMockFileSystem(iterator); + + ParquetDataManager manager = new ParquetDataManager(CONF, "test-path", mockFs); + + ParquetFileInfo result = manager.getParquetDataFileAt(2000L); + + assertNotNull(result); + assertEquals(2000L, result.getModificationTime()); + } + + @Test + void testGetParquetDataFileAt_firstAfterTime() throws IOException { + LocatedFileStatus file1 = createMockFileStatus("file1.parquet", 1000L); + LocatedFileStatus file2 = createMockFileStatus("file2.parquet", 2500L); + LocatedFileStatus file3 = createMockFileStatus("file3.parquet", 3000L); + + RemoteIterator<LocatedFileStatus> iterator = + new StubbedRemoteIterator(Arrays.asList(file1, file2, file3)); + FileSystem mockFs = createMockFileSystem(iterator); + + ParquetDataManager manager = new ParquetDataManager(CONF, "test-path", mockFs); + + ParquetFileInfo result = manager.getParquetDataFileAt(2000L); + + assertNotNull(result); + assertEquals(2500L, result.getModificationTime()); + } + + @Test + void testGetParquetDataFileAt_multipleAfterTime() throws IOException { + LocatedFileStatus file1 = createMockFileStatus("file1.parquet", 1000L); + LocatedFileStatus file2 = createMockFileStatus("file2.parquet", 2500L); + LocatedFileStatus file3 = createMockFileStatus("file3.parquet", 3000L); + + RemoteIterator<LocatedFileStatus> iterator = + new StubbedRemoteIterator(Arrays.asList(file1, file2, file3)); + FileSystem mockFs = createMockFileSystem(iterator); + + ParquetDataManager manager = new ParquetDataManager(CONF, "test-path", mockFs); + + ParquetFileInfo result = manager.getParquetDataFileAt(2000L); + + assertNotNull(result); + assertEquals(2500L, result.getModificationTime()); + } + + @Test + void testGetParquetDataFileAt_noMatch() throws IOException { + RemoteIterator<LocatedFileStatus> iterator = new StubbedRemoteIterator(Collections.emptyList()); + FileSystem mockFs = createMockFileSystem(iterator); + + ParquetDataManager manager = new ParquetDataManager(CONF, "test-path", mockFs); + + IllegalStateException exception = + assertThrows(IllegalStateException.class, () -> manager.getParquetDataFileAt(5000L)); + assertTrue(exception.getMessage().contains("No file found at or after 5000")); + } + + @Test + void testGetParquetDataFileAt_allBefore() throws IOException { + LocatedFileStatus file1 = createMockFileStatus("file1.parquet", 1000L); + LocatedFileStatus file2 = createMockFileStatus("file2.parquet", 2000L); + + RemoteIterator<LocatedFileStatus> iterator = + new StubbedRemoteIterator(Arrays.asList(file1, file2)); + FileSystem mockFs = createMockFileSystem(iterator); + + ParquetDataManager manager = new ParquetDataManager(CONF, "test-path", mockFs); + + IllegalStateException exception = + assertThrows(IllegalStateException.class, () -> manager.getParquetDataFileAt(3000L)); + assertTrue(exception.getMessage().contains("No file found at or after 3000")); + } + + @Test + void testGetCurrentFilesInfo_multipleFiles() throws IOException { + LocatedFileStatus file1 = createMockFileStatus("file1.parquet", 1000L); + LocatedFileStatus file2 = createMockFileStatus("file2.parquet", 2000L); + LocatedFileStatus file3 = createMockFileStatus("file3.parquet", 3000L); + + RemoteIterator<LocatedFileStatus> iterator = + new StubbedRemoteIterator(Arrays.asList(file1, file2, file3)); + FileSystem mockFs = createMockFileSystem(iterator); + + ParquetDataManager manager = new ParquetDataManager(CONF, "test-path", mockFs); + + Stream<ParquetFileInfo> result = manager.getCurrentFilesInfo(); + + assertNotNull(result); + List<ParquetFileInfo> fileList = result.collect(Collectors.toList()); + assertEquals(3, fileList.size()); + assertEquals(1000L, fileList.get(0).getModificationTime()); Review Comment: I'm updating the test since sorting in the API requires you to pull everything into memory before sorting. ########## xtable-core/src/test/java/org/apache/xtable/parquet/ITParquetConversionSource.java: ########## @@ -418,6 +432,124 @@ private void checkDatasetEquivalence( } } + @Test + void testIncrementalSyncWithMultiplePartitions() throws IOException { + + Configuration conf = sparkSession.sparkContext().hadoopConfiguration(); + + StructType schema = + DataTypes.createStructType( + new StructField[] { + DataTypes.createStructField("id", DataTypes.IntegerType, false), + DataTypes.createStructField("value", DataTypes.StringType, false), + DataTypes.createStructField("year", DataTypes.IntegerType, false), + DataTypes.createStructField("month", DataTypes.IntegerType, false) + }); + List<Row> data = + Arrays.asList( + RowFactory.create(100, "A", 2026, 12), + RowFactory.create(101, "AA", 2026, 12), + RowFactory.create(102, "CB", 2027, 11), + RowFactory.create(103, "BA", 2027, 11)); + + Dataset<Row> dfInit = sparkSession.createDataFrame(data, schema); + Path fixedPath = Paths.get("target", "fixed-parquet-data", "parquet_table_test_2"); Review Comment: Updated this -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
