hsiang-c commented on code in PR #4752: URL: https://github.com/apache/datafusion-comet/pull/4752#discussion_r3680212697
########## spark/src/test/resources/sql-tests/iceberg/metadata_column_partition.sql: ########## @@ -0,0 +1,850 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Native Iceberg scan is unsupported on Spark 4.2: no Iceberg spark-runtime is published for +-- 4.2 yet, so the build reuses the 4.0 runtime, which is binary-incompatible with 4.2 (e.g. +-- connector.catalog.View became a class). Mirrors the assume(!isSpark42Plus) guard in +-- CometIcebergNativeSuite. See https://github.com/apache/datafusion-comet/issues/4969. +-- MaxSparkVersion: 4.1 + +-- Config: spark.sql.catalog.test_cat=org.apache.iceberg.spark.SparkCatalog +-- Config: spark.sql.catalog.test_cat.type=hadoop +-- Config: spark.sql.catalog.test_cat.warehouse=/tmp/comet-iceberg-sql-test +-- Config: spark.comet.enabled=true +-- Config: spark.comet.exec.enabled=true +-- Config: spark.comet.iceberg.native.enabled=true + +-- All `query` assertions below implicitly validate that Comet's partition type +-- computation (from partition_type_pool JSON) matches Spark's (from Iceberg Java's +-- PartitionSpec.partitionType()). If the two diverged, checkSparkAnswerAndOperator +-- would detect differing struct values. This covers the "dual computation path" +-- cross-validation concern. + +-- ============================================================= +-- Setup: Create table with bucket partitioning (format v2 for MoR) +-- ============================================================= + +statement +DROP TABLE IF EXISTS test_cat.db.meta_test + +statement +CREATE TABLE test_cat.db.meta_test (id INT, name STRING, value DOUBLE) USING iceberg PARTITIONED BY (bucket(4, id)) TBLPROPERTIES ('format-version' = '2') + +statement +INSERT INTO test_cat.db.meta_test VALUES (1, 'alice', 10.0), (2, 'bob', 20.0), (3, 'charlie', 30.0) + +statement +INSERT INTO test_cat.db.meta_test VALUES (4, 'dave', 40.0), (5, 'eve', 50.0) + +-- ============================================================= +-- _file: basic projection (native) +-- ============================================================= + +query +SELECT id, _file FROM test_cat.db.meta_test ORDER BY id + +-- Multiple inserts produce multiple files +query +SELECT COUNT(DISTINCT _file) > 1 FROM test_cat.db.meta_test + +-- _file in a WHERE clause exercises filter pushdown interaction +query +SELECT id, name FROM test_cat.db.meta_test WHERE _file LIKE '%.parquet' ORDER BY id + +-- ============================================================= +-- _partition: basic struct projection (native) +-- ============================================================= + +query +SELECT id, _partition FROM test_cat.db.meta_test ORDER BY id + +-- Access individual partition fields within the struct +query +SELECT id, _partition.id_bucket FROM test_cat.db.meta_test ORDER BY id + +-- ============================================================= +-- _spec_id: partition spec ID (scalar per file) +-- ============================================================= + +query +SELECT id, _spec_id FROM test_cat.db.meta_test ORDER BY id + +-- All rows should have spec_id = 0 (initial spec) +query +SELECT DISTINCT _spec_id FROM test_cat.db.meta_test + +-- ============================================================= +-- _pos: row position within each file (0-based) +-- ============================================================= + +-- Basic: positions should be 0-based within each file +query +SELECT id, _file, _pos FROM test_cat.db.meta_test ORDER BY _file, _pos + +-- Positions should be contiguous 0..N-1 per file +query +SELECT _file, MIN(_pos) as min_pos, MAX(_pos) as max_pos, COUNT(*) as cnt FROM test_cat.db.meta_test GROUP BY _file + +-- ============================================================= +-- Partition evolution requires IcebergSparkSessionExtensions +-- (ALTER TABLE ADD PARTITION FIELD / CALL system.add_partition_field). +-- The SQL file test framework cannot register session extensions via +-- Config directives. Partition evolution is covered in +-- CometIcebergNativeSuite which has full session control. Review Comment: Awesome! -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
