gustavoatt commented on issue #2976:
URL: https://github.com/apache/iceberg/issues/2976#issuecomment-1006097589
I'm seeing something similar only with Spark 3.1.x (it works fine on Spark
2.4.x and Spark 3.2.x) from my tests. Here is a repro I was able to create:
```sql
CREATE TABLE test_table (
`meta_data` STRUCT<`event_schema`: STRING, `id`: STRING, `tiers`:
ARRAY<STRUCT<`id`: STRING, `type`: STRING, `hostname`: STRING, `ip_address`:
STRING, `timestamp`: BIGINT, `sequences`: ARRAY<STRUCT<`key`: ARRAY<STRING>,
`value`: BIGINT, `bytes_sent`: BIGINT>>>>, `kafka_metadata`: STRUCT<`topic`:
STRING, `partition`: INT, `offset`: BIGINT>, `properties`: MAP<STRING, STRING>,
`message_type`: STRING>,
`schema_version` STRING,
`context_user_id` BIGINT,
`context_visitor_id` STRING,
`schema` STRING,
`event_name` STRING,
`context` STRUCT<`timestamp`: BIGINT, `source`: STRING, `platform`:
STRING, `user_agent`: STRING, `version`: STRING, `bev`: STRING, `user_id`:
BIGINT, `hash_user_id`: STRING, `language`: STRING, `locale`: STRING,
`campaign`: STRING, `affiliate_id`: STRING, `screen_width`: BIGINT,
`screen_height`: BIGINT, `web`: STRUCT<`controller`: STRING, `page`: STRING,
`action`: STRING, `rendered_on`: STRING, `viewport`: STRING, `req_uuid`:
STRING, `page_uri`: STRING, `page_referrer`: STRING, `canonical_url`: STRING,
`shardset`: STRING, `local_time`: STRING, `queued`: BOOLEAN, `canonical_host`:
STRING, `app`: STRING, `section_id`: STRING>, `monorail`: STRUCT<`controller`:
STRING, `action`: STRING, `api_client_id`: STRING, `api_device_id`: STRING,
`api_client_name`: STRING, `api_protocol`: STRING, `api_version`: STRING,
`api_resource`: STRING, `api_format`: STRING, `api_intents`: STRING>, `mobile`:
STRUCT<`device_type`: STRING, `device_id`: STRING, `version_code`: BIGINT,
`screen_orientation
`: STRING, `network_type`: STRING, `carrier_name`: STRING, `carrier_country`:
STRING, `build_type`: STRING, `view_class_name`: STRING, `native_mode`: STRING,
`module_name`: STRING>, `extra_data`: MAP<STRING, STRING>, `client_session_id`:
STRING, `bev_expiry`: STRING, `service_context`: STRUCT<`service_name`: STRING,
`local_request_id`: STRING, `http_method_type`: STRING, `endpoint_name`:
STRING, `global_request_chain_id`: STRING>, `page_name`: STRING, `amp_id`:
STRING, `req_remote_host`: STRING, `amp_client_session_id`: STRING,
`previous_page_name`: STRING, `visitor_id`: STRING, `impression_uuid`: STRING,
`api_request_id`: STRING, `max_deferred_minutes`: BIGINT, `environment`:
STRING, `device`: STRUCT<`device_year_class`: STRING>, `traffic_type`: STRING,
`context_api_client_id`: STRING>,
`experiment` STRING,
`treatment` STRING,
`subject_type` STRING,
`subject_id` STRING,
`client` STRING,
`client_version` STRING,
`user_id` STRING,
`visitor_id` STRING,
`misa_id` STRING,
`listing_id` STRING,
`extra_data` MAP<STRING, STRING>,
`ds` STRING,
`hr` STRING
)
USING iceberg
PARTITIONED BY (ds, hr);
-- Insert only nulls (but error happens even with non-null columns)
INSERT INTO test_table (
meta_data,
schema_version,
context_user_id,
context_visitor_id,
schema,
event_name,
context,
experiment,
treatment,
subject_type,
subject_id,
client,
client_version,
user_id,
visitor_id,
misa_id,
listing_id,
extra_data,
ds,
hr
)
VALUES (
null, null, null, null, null, null, null, null, null, null, null, null,
null, null, null, null, null, null, '2022-01-01', '10'
);
-- If we select everything we have no issue whatsoever.
SELECT *
FROM test_table;
-- However, if we try to filter based on any column we get a class cast
exception, even though the results should be the
-- same.
SELECT *
FROM test_table
WHERE hr = '10';
-- The same query above works if we select fewer columns. In this case
selecting < 14 columns works correctly
select
meta_data,
schema_version,
context_user_id,
context_visitor_id,
schema,
event_name,
context,
experiment,
treatment,
subject_type,
subject_id,
client,
client_version,
user_id,
hr
from test_table
WHERE hr = '10';
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
