Muneer320 commented on code in PR #27151:
URL: https://github.com/apache/datafusion/pull/27151#discussion_r4619110264
##########
datafusion/catalog-listing/src/helpers.rs:
##########
@@ -594,6 +611,30 @@ mod tests {
vec!["mypartition"]
)
);
+ assert_eq!(
+ None,
+ parse_partitions_for_path(
+ &ListingTableUrl::parse("file:///bucket/mytable").unwrap(),
+ &Path::from("bucket/mytable/mypartition=v1.csv"),
+ vec!["mypartition"]
+ )
+ );
+ assert_eq!(
+ None,
+ parse_partitions_for_path(
+ &ListingTableUrl::parse("file:///bucket/mytable").unwrap(),
+
&Path::from("bucket/mytable/mypartition=v1/otherpartition=v2.csv"),
+ vec!["mypartition", "otherpartition"]
+ )
+ );
Review Comment:
Thanks, I’ve added the three-column case as a separate regression so it’s
checked independently of the existing two-column case.
##########
datafusion/sqllogictest/test_files/listing_table_partitions.slt:
##########
@@ -257,3 +257,81 @@ select a, p from null_part_write order by a;
statement ok
set datafusion.execution.listing_table_hive_default_partition_name =
'__HIVE_DEFAULT_PARTITION__';
+
+# A root filename cannot supply an integer partition value. Previously this
+# attempted to cast "2025.parquet" to INT and failed the query; it is now
skipped.
Review Comment:
Updated the comment to describe the current behavior.
##########
datafusion/sqllogictest/test_files/listing_table_partitions.slt:
##########
@@ -257,3 +257,81 @@ select a, p from null_part_write order by a;
statement ok
set datafusion.execution.listing_table_hive_default_partition_name =
'__HIVE_DEFAULT_PARTITION__';
+
+# A root filename cannot supply an integer partition value. Previously this
+# attempted to cast "2025.parquet" to INT and failed the query; it is now
skipped.
+query I
+copy (values(7))
+to
'test_files/scratch/listing_table_partitions/filename_partition_int/year=2024/data.parquet';
+----
+1
+
+statement ok
+create external table filename_partition_int (column1 bigint)
+partitioned by (year int)
+stored as parquet
+location 'test_files/scratch/listing_table_partitions/filename_partition_int/';
+
+# A new stray file arrives after the table's valid layout has been registered.
+query I
+copy (values(8))
+to
'test_files/scratch/listing_table_partitions/filename_partition_int/year=2025.parquet';
+----
+1
+
+statement ok
+set datafusion.runtime.list_files_cache_limit = '0K';
+
+query II
+select column1, year from filename_partition_int order by column1;
+----
+7 2024
+
+# A filename shaped like a partition must not supply a missing directory level.
+query I
+copy (values(3))
+to
'test_files/scratch/listing_table_partitions/filename_partition/year=2024/month=01/a.parquet';
+----
+1
+
+statement ok
+create external table filename_partition (column1 bigint)
+partitioned by (year varchar, month varchar)
+stored as parquet
+location 'test_files/scratch/listing_table_partitions/filename_partition/';
+
+query I
+copy (values(4))
+to
'test_files/scratch/listing_table_partitions/filename_partition/year=2024/month=02.parquet';
+----
+1
+
+query I
+copy (values(5))
+to
'test_files/scratch/listing_table_partitions/filename_partition/year=2024/data.parquet';
+----
+1
+
+# A key=value filename inside a complete partition path remains a valid file.
+query I
+copy (values(6))
+to
'test_files/scratch/listing_table_partitions/filename_partition/year=2024/month=01/month=03.parquet';
+----
+1
+
+statement ok
+set datafusion.runtime.list_files_cache_limit = '0K';
Review Comment:
Good point, the first SET already covers both fixtures. I’ve removed the
duplicate and added the runtime cache RESET at the end, before restoring the
catalog setting.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]