Asura7969 commented on code in PR #8565:
URL: https://github.com/apache/arrow-datafusion/pull/8565#discussion_r1433335906
##########
datafusion/sqllogictest/test_files/csv_files.slt:
##########
@@ -63,3 +63,60 @@ id6 value"6
id7 value"7
id8 value"8
id9 value"9
+
+
+# When reading a partitioned table, the `listing_table_ignore_subdirectory`
configuration will be invalid
+statement ok
+set datafusion.execution.listing_table_ignore_subdirectory = false;
+
+statement ok
+CREATE EXTERNAL TABLE partition_csv_table (
+ name VARCHAR,
+ ts TIMESTAMP,
+ c_date DATE,
+)
+STORED AS CSV
+PARTITIONED BY (c_date)
+LOCATION '../core/tests/data/partitioned_table';
+
+query I
+select count(*) from partition_csv_table;
+----
+4
+
+statement ok
+DROP TABLE partition_csv_table
+
+statement ok
+set datafusion.execution.listing_table_ignore_subdirectory = true;
+
+statement ok
+CREATE EXTERNAL TABLE partition_csv_table (
+ name VARCHAR,
+ ts TIMESTAMP,
+ c_date DATE,
+)
+STORED AS CSV
+PARTITIONED BY (c_date)
+LOCATION '../core/tests/data/partitioned_table';
+
+query TT
+explain select count(*) from partition_csv_table;
+----
+logical_plan
+Aggregate: groupBy=[[]], aggr=[[COUNT(UInt8(1)) AS COUNT(*)]]
+--TableScan: partition_csv_table projection=[]
+physical_plan
+AggregateExec: mode=Final, gby=[], aggr=[COUNT(*)]
+--CoalescePartitionsExec
+----AggregateExec: mode=Partial, gby=[], aggr=[COUNT(*)]
+------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2
+--------CsvExec: file_groups={2 groups:
[[WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_table/c_date=2018-11-13/timestamps.csv],
[WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_table/c_date=2018-12-13/timestamps.csv]]},
has_header=false
+
+query I
+select count(*) from partition_csv_table;
Review Comment:
When reading a partitioned table, `listing_table_ignore_subdirectory` is
always equal to false, even if the default is true
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
