tkalkirill commented on code in PR #5305: URL: https://github.com/apache/calcite/pull/5305#discussion_r4207663338
########## core/src/test/resources/sql/row-equality.iq: ########## @@ -612,4 +612,93 @@ ORDER BY c; !ok +# [CALCITE-7836] Array functions compare ROW elements by field values. +# Construct equal rows from separate input columns so constant folding +# and expression reuse do not mask reference equality bugs. +!use sparkfunc + +# Flat ROW values. +SELECT ARRAY_CONTAINS(ARRAY[a], b) AS contains_row, Review Comment: Split the test into separate functions and cases. I had the check-up done here: https://www.sparkplayground.com/pyspark-online-compiler ``` def run_validation(display): from pyspark.sql import SparkSession, Row spark = SparkSession.builder.appName('CALCITE-7836 full results').getOrCreate() print('Apache Spark version:', spark.version) # Equivalent to the 32 CALCITE-7836 Quidem queries. # ARRAY[...] becomes array(...); ROW(...) becomes named_struct(...). cases = [ ('flat', "named_struct('id', x, 'label', y)", "named_struct('id', z, 'label', w)", (1, 'Foo'), (2, 'Bar'), (2, 'Car')), ('nested_row', "named_struct('r', named_struct('id', x, 'label', y), 'n', 10)", "named_struct('r', named_struct('id', z, 'label', w), 'n', 10)", ((1, 'Foo'), 10), ((2, 'Bar'), 10), ((2, 'Car'), 10)), ('array_in_row', "named_struct('r', array(named_struct('id', x, 'label', y)), 'n', 10)", "named_struct('r', array(named_struct('id', z, 'label', w)), 'n', 10)", ([(1, 'Foo')], 10), ([(2, 'Bar')], 10), ([(2, 'Car')], 10)), ('null_field', "named_struct('id', x, 'v', cast(null as int))", "named_struct('id', z, 'v', cast(null as int))", (1, None), (2, None), (2, None)) ] def normalize(value): if isinstance(value, Row): return tuple(normalize(v) for v in value) if isinstance(value, list): return [normalize(v) for v in value] return value def format_value(value): if value is None: return 'null' if isinstance(value, bool): return str(value).lower() if isinstance(value, tuple): return '{' + ', '.join(format_value(v) for v in value) + '}' if isinstance(value, list): return '[' + ', '.join(format_value(v) for v in value) + ']' return str(value) summary = [] for name, a, b, first, left, right in cases: equal = left == right checks = [ ('ARRAY_CONTAINS', f'array_contains(array({a}), {b})', [True, equal]), ('ARRAY_POSITION', f'array_position(array({a}), {b})', [1, 1 if equal else 0]), ('ARRAY_REMOVE', f'array_remove(array({a}, {b}), {b})', [[], [] if equal else [left]]), ('ARRAY_DISTINCT', f'array_distinct(array({a}, {b}))', [[first], [left] if equal else [left, right]]), ('ARRAY_UNION', f'array_union(array({a}), array({b}))', [[first], [left] if equal else [left, right]]), ('ARRAY_INTERSECT', f'array_intersect(array({a}), array({b}))', [[first], [left] if equal else []]), ('ARRAY_EXCEPT', f'array_except(array({a}), array({b}))', [[], [] if equal else [left]]), ('ARRAYS_OVERLAP', f'arrays_overlap(array({a}), array({b}))', [True, equal]) ] for function, expression, expected in checks: query = f"SELECT {expression} AS r FROM VALUES (1, 'Foo', 1, 'Foo'), (2, 'Bar', 2, 'Car') AS t(x, y, z, w) ORDER BY x" actual = [normalize(row.r) for row in spark.sql(query).collect()] assert actual == expected, f'{name} {function}: expected {expected}, got {actual}' summary.append((name, function, format_value(actual[0]), format_value(actual[1]), 'PASS')) print(name + ': PASS (8 queries, 16 full results)') print('ALL 32 QUERIES / 64 FULL RESULTS MATCH.') for name, function, first, second, status in summary: print(f'{name} {function}: {first} | {second} [{status}]') display(spark.createDataFrame(summary, ['scenario', 'function', 'first_result', 'second_result', 'status'])) run_validation(display) ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
