chamikaramj commented on code in PR #39758:
URL: https://github.com/apache/beam/pull/39758#discussion_r3833372385
##########
sdks/java/io/delta/src/test/java/org/apache/beam/sdk/io/delta/DeltaIOIT.java:
##########
@@ -302,6 +302,116 @@ public void testReadDeltaLakeTable() {
readPipeline.run().waitUntilFinish();
}
+ @Test
Review Comment:
It's included in
https://github.com/apache/beam/blob/master/.github/workflows/beam_PostCommit_Java_Delta_IO_Dataflow.yml.
Executed for the current PR here:
https://github.com/apache/beam/actions/runs/31854212817/job/94935775426?pr=39758
##########
sdks/java/io/delta/src/test/java/org/apache/beam/sdk/io/delta/DeltaIOIT.java:
##########
@@ -302,6 +302,116 @@ public void testReadDeltaLakeTable() {
readPipeline.run().waitUntilFinish();
}
+ @Test
+ public void testReadDeltaLakeTableAtTimestamp() throws Exception {
+ ExperimentalOptions options =
readPipeline.getOptions().as(ExperimentalOptions.class);
+ ExperimentalOptions.addExperiment(options, "use_runner_v2");
+
+ Map<String, String> hadoopConfig = new HashMap<>();
+ hadoopConfig.put("fs.gs.impl",
"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem");
Review Comment:
Done.
##########
sdks/java/io/delta/src/test/java/org/apache/beam/sdk/io/delta/DeltaIOIT.java:
##########
@@ -302,6 +302,116 @@ public void testReadDeltaLakeTable() {
readPipeline.run().waitUntilFinish();
}
+ @Test
+ public void testReadDeltaLakeTableAtTimestamp() throws Exception {
+ ExperimentalOptions options =
readPipeline.getOptions().as(ExperimentalOptions.class);
+ ExperimentalOptions.addExperiment(options, "use_runner_v2");
+
+ Map<String, String> hadoopConfig = new HashMap<>();
+ hadoopConfig.put("fs.gs.impl",
"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem");
+ hadoopConfig.put(
+ "fs.AbstractFileSystem.gs.impl",
"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS");
+ hadoopConfig.put("fs.gs.auth.type", "APPLICATION_DEFAULT");
+ String project =
+ readPipeline
+ .getOptions()
+ .as(org.apache.beam.sdk.extensions.gcp.options.GcpOptions.class)
+ .getProject();
+ if (project != null) {
+ hadoopConfig.put("fs.gs.project.id", project);
+ }
+
+ org.apache.hadoop.conf.Configuration conf = new
org.apache.hadoop.conf.Configuration();
+ for (Map.Entry<String, String> entry : hadoopConfig.entrySet()) {
+ conf.set(entry.getKey(), entry.getValue());
+ }
+ Engine engine = DefaultEngine.create(conf);
+
+ // Wait briefly to ensure timestamp is after version 0 commit
+ Thread.sleep(1000);
Review Comment:
Done.
##########
sdks/java/io/delta/src/test/java/org/apache/beam/sdk/io/delta/DeltaIOTest.java:
##########
@@ -109,6 +109,185 @@ public void testReadRowsNullDefaults() {
Assert.assertNull(readRows.getHadoopConfig());
}
+ @Test
+ public void testReadRowsBothVersionAndTimestampThrows() {
+ org.apache.beam.sdk.Pipeline p = org.apache.beam.sdk.Pipeline.create();
+ IllegalArgumentException exception =
+ Assert.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ p.apply(
+ DeltaIO.readRows()
+ .from("/path/to/table")
+ .withVersion(0L)
+ .withTimestamp("2026-05-20T15:43:26Z")));
+ Assert.assertTrue(exception.getMessage().contains("Cannot set both version
and timestamp."));
+ }
+
+ @Test
+ public void testReadRowsAtVersion() throws Exception {
+ File tableDir = tempFolder.newFolder("delta-table-read-version");
+ Engine engine = DefaultEngine.create(new
org.apache.hadoop.conf.Configuration());
+
+ Schema schema = Schema.builder().addField("name",
Schema.FieldType.STRING).build();
Review Comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
