sapienza88 commented on code in PR #768:
URL: https://github.com/apache/incubator-xtable/pull/768#discussion_r3113525180
##########
xtable-core/src/test/java/org/apache/xtable/parquet/ITParquetConversionSource.java:
##########
@@ -418,6 +432,124 @@ private void checkDatasetEquivalence(
}
}
+ @Test
+ void testIncrementalSyncWithMultiplePartitions() throws IOException {
+
+ Configuration conf = sparkSession.sparkContext().hadoopConfiguration();
+
+ StructType schema =
+ DataTypes.createStructType(
+ new StructField[] {
+ DataTypes.createStructField("id", DataTypes.IntegerType, false),
+ DataTypes.createStructField("value", DataTypes.StringType,
false),
+ DataTypes.createStructField("year", DataTypes.IntegerType,
false),
+ DataTypes.createStructField("month", DataTypes.IntegerType,
false)
+ });
+ List<Row> data =
+ Arrays.asList(
+ RowFactory.create(100, "A", 2026, 12),
+ RowFactory.create(101, "AA", 2026, 12),
+ RowFactory.create(102, "CB", 2027, 11),
+ RowFactory.create(103, "BA", 2027, 11));
+
+ Dataset<Row> dfInit = sparkSession.createDataFrame(data, schema);
+ Path fixedPath = Paths.get("target", "fixed-parquet-data",
"parquet_table_test_2");
+ // String outputPath = fixedPath.toString();
+ Dataset<Row> df = dfInit.withColumn("full_date", expr("make_date(year,
month, 1)"));
+ String outputPath =
+ new
java.io.File("target/fixed-parquet-data/parquet_table_test_2").getAbsolutePath();
+ df.coalesce(1).write().partitionBy("year",
"month").mode("overwrite").parquet(outputPath);
+
+ // test find files to sync
+ FileSystem fs = FileSystem.get(fixedPath.toUri(), conf);
+ // set the modification time to the table file
+ // update modificationTime for file to append
+ // many partitions case
+ List<String> newPartitions = Arrays.asList("year=2026/month=12",
"year=2027/month=11");
+ long targetModificationTime = System.currentTimeMillis() - 360000;
+ long newModificationTime = System.currentTimeMillis() - 50000;
+ long testTime = System.currentTimeMillis() - 90000; // between two prev
times
+ for (String partition : newPartitions) {
+ org.apache.hadoop.fs.Path partitionPath =
+ new org.apache.hadoop.fs.Path(outputPath, partition);
+ if (fs.exists(partitionPath)) {
+ updateModificationTimeRecursive(fs, partitionPath,
targetModificationTime);
+ }
+ }
+ // create new file to append using Spark
+ List<Row> futureDataToSync =
+ Arrays.asList(
+ RowFactory.create(101, "A", 2026, 12),
+ RowFactory.create(301, "D", 2027, 11),
+ RowFactory.create(302, "DA", 2027, 11));
+ Dataset<Row> dfToSyncInit = sparkSession.createDataFrame(futureDataToSync,
schema);
+ Dataset<Row> dfToSync = dfToSyncInit.withColumn("full_date",
expr("make_date(year, month, 1)"));
+ dfToSync.coalesce(1).write().partitionBy("year",
"month").mode("append").parquet(outputPath);
+
+ // conversionSource operations
+ Properties sourceProperties = new Properties();
+ String partitionConfig = "full_date:MONTH:year=yyyy/month=MM";
+ sourceProperties.put(PARTITION_FIELD_SPEC_CONFIG, partitionConfig);
+
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name("parquet_table_test_2")
+ .basePath(fixedPath.toAbsolutePath().toString())
+ .additionalProperties(sourceProperties)
+ .formatName(TableFormat.PARQUET)
+ .build();
+
+ ParquetConversionSourceProvider conversionSourceProvider =
+ new ParquetConversionSourceProvider();
+ conversionSourceProvider.init(conf);
+ ParquetConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+
+ for (String partition : newPartitions) {
+ org.apache.hadoop.fs.Path partitionPath =
+ new org.apache.hadoop.fs.Path(outputPath, partition);
+
+ RemoteIterator<LocatedFileStatus> it = fs.listFiles(partitionPath,
false);
+ while (it.hasNext()) {
+ LocatedFileStatus fileStatus = it.next();
+
+ if (fileStatus.getModificationTime() > newModificationTime) {
+ fs.setTimes(fileStatus.getPath(), newModificationTime, -1);
+ } else {
+ fs.setTimes(fileStatus.getPath(), targetModificationTime, -1);
+ }
+ }
+ fs.setTimes(partitionPath, newModificationTime, -1);
+ }
+
+ InternalTable result = conversionSource.getTable(newModificationTime);
+ assertEquals(
+ Instant.ofEpochMilli(newModificationTime).toString(),
+ result.getLatestCommitTime().toString());
+ assertEquals("parquet_table_test_2", result.getName());
+ assertEquals(TableFormat.PARQUET, result.getTableFormat());
+ assertNotNull(result.getReadSchema());
+ InternalSnapshot snapshot = conversionSource.getCurrentSnapshot();
+ assertNotNull(snapshot);
+ TableChange changes =
conversionSource.getTableChangeForCommit(newModificationTime);
+ assertNotNull(changes);
+ Instant instantBeforeFirstSnapshot =
+
Instant.ofEpochMilli(snapshot.getTable().getLatestCommitTime().toEpochMilli());
+ assertEquals(instantBeforeFirstSnapshot.toEpochMilli(),
newModificationTime);
+
assertTrue(conversionSource.isIncrementalSyncSafeFrom(Instant.ofEpochMilli(testTime)));
Review Comment:
added a test here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]