vinishjail97 commented on code in PR #801:
URL: https://github.com/apache/incubator-xtable/pull/801#discussion_r3331832679


##########
xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelSchemaExtractor.java:
##########
@@ -873,4 +883,142 @@ public void testIcebergToDeltaUUIDSupport() {
         internalSchema,
         
DeltaKernelSchemaExtractor.getInstance().toInternalSchema(structRepresentation));
   }
+
+  // ========== Tests for fromInternalSchema() - New Tests ==========

Review Comment:
   Addressed — the `New Tests` comment has been removed.



##########
xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelSchemaExtractor.java:
##########
@@ -873,4 +883,142 @@ public void testIcebergToDeltaUUIDSupport() {
         internalSchema,
         
DeltaKernelSchemaExtractor.getInstance().toInternalSchema(structRepresentation));
   }
+
+  // ========== Tests for fromInternalSchema() - New Tests ==========
+
+  @Test

Review Comment:
   Addressed — `TestDeltaKernelSchemaExtractor` now covers nested records as 
well as arrays (`ArrayType`) and maps (`MapType`).



##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileUpdatesExtractor.java:
##########
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+package org.apache.xtable.kernel;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import lombok.Builder;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.MapValue;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.util.VectorUtils;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FilesDiff;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalFile;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.paths.PathUtils;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+@Builder
+public class DeltaKernelDataFileUpdatesExtractor {
+  @Builder.Default
+  private final DeltaKernelStatsExtractor deltaStatsExtractor =
+      DeltaKernelStatsExtractor.getInstance();
+
+  @Builder.Default
+  private final DeltaKernelPartitionExtractor deltaKernelPartitionExtractor =
+      DeltaKernelPartitionExtractor.getInstance();
+
+  @Builder.Default
+  private final DeltaKernelDataFileExtractor dataFileExtractor =
+      DeltaKernelDataFileExtractor.builder().build();
+
+  private final Engine engine;
+  private final String basePath;
+  private final boolean includeColumnStats;
+
+  public Seq<RowBackedAction> applySnapshot(
+      Table table, List<PartitionFileGroup> partitionedDataFiles, 
InternalSchema tableSchema) {
+
+    // all files in the current delta snapshot are potential candidates for 
remove actions, i.e. if
+    // the file is not present in the new snapshot (addedFiles) then the file 
is considered removed
+    Map<String, RowBackedAction> previousFiles = new HashMap<>();
+    StructType physicalSchema;
+
+    // Check if table exists by checking if _delta_log directory exists
+    boolean tableExists = checkTableExists(table);
+
+    if (tableExists) {
+      Snapshot snapshot = table.getLatestSnapshot(engine);
+
+      // Reuse DeltaKernelDataFileExtractor to iterate through existing files
+      // This avoids duplicating the scan logic for reading Delta files
+      try (DataFileIterator fileIterator =
+          dataFileExtractor.iterator(snapshot, table, engine, tableSchema)) {
+
+        while (fileIterator.hasNext()) {
+          InternalDataFile internalFile = fileIterator.next();
+
+          // Convert InternalDataFile back to AddFile to create RemoveFile 
action
+          AddFile addFile = createAddFileFromInternalDataFile(internalFile, 
snapshot.getSchema());
+          RemoveFile removeFile =
+              new RemoveFile(addFile.toRemoveFileRow(false, 
Optional.of(snapshot.getVersion())));
+          String fullPath =
+              
DeltaKernelActionsConverter.getFullPathToFile(removeFile.getPath(), table);
+          previousFiles.put(fullPath, removeFile);
+        }
+      } catch (Exception e) {
+        throw new ReadException("Failed to scan existing Delta files", e);
+      }
+
+      physicalSchema = snapshot.getSchema();
+
+    } else {
+
+      // Table doesn't exist yet - no previous files to remove
+      // Convert InternalSchema to StructType for physical schema
+      DeltaKernelSchemaExtractor schemaExtractor = 
DeltaKernelSchemaExtractor.getInstance();
+      physicalSchema = schemaExtractor.fromInternalSchema(tableSchema);
+    }
+
+    FilesDiff<InternalFile, RowBackedAction> diff =
+        InternalFilesDiff.findNewAndRemovedFiles(partitionedDataFiles, 
previousFiles);
+
+    return applyDiff(
+        diff.getFilesAdded(),
+        diff.getFilesRemoved(),
+        tableSchema,
+        table.getPath(engine),
+        physicalSchema);
+  }
+
+  private boolean checkTableExists(Table table) {
+    return DeltaKernelUtils.tableExists(engine, table.getPath(engine));
+  }
+
+  /**
+   * Converts an InternalDataFile back to Delta Kernel's AddFile action. This 
is needed to create
+   * RemoveFile actions from existing files.
+   */
+  private AddFile createAddFileFromInternalDataFile(

Review Comment:
   Addressed — there is now a single `createAddFileAction` reused for both the 
add and remove paths.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to