Gabriel39 commented on code in PR #66321: URL: https://github.com/apache/doris/pull/66321#discussion_r3691148600
########## regression-test/suites/paimon_write/test_paimon_write_variant.groovy: ########## @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_variant", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_variant_catalog" + String dbName = "test_pw_variant_db" + String root = '$' + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_basic; + CREATE TABLE paimon.${dbName}.t_variant_basic ( + id INT, + payload VARIANT, + secondary VARIANT + ) USING paimon + TBLPROPERTIES ('file.format' = 'parquet'); + """ + + sql """DROP CATALOG IF EXISTS ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """SWITCH ${catalogName}""" + sql """USE ${dbName}""" + + try { + // Paimon Variant writes are deliberately V2-only. + sql """SET enable_variant_v2 = false""" + test { + sql """INSERT INTO t_variant_basic VALUES + (0, parse_to_variant('{"disabled":true}'), NULL)""" + exception "set enable_variant_v2=true" + } + sql """SET enable_variant_v2 = true""" + + // JSON containers, JSON null and SQL NULL are different logical values. + sql """ + INSERT INTO t_variant_basic VALUES + (1, parse_to_variant(CONCAT( + '{"object":{"name":"doris","address":{"city":"Hangzhou"}},"array":[1,true,null,"x"],"emptyObject":{},"emptyArray":[],"explicitNull":null,"escaped":"line', + CHAR(92), 'nquote', CHAR(92), CHAR(34), '","unicode":"δΈζπ"}')), + parse_to_variant('{"second":2}')), + (2, parse_to_variant('{}'), parse_to_variant('[]')), + (3, parse_to_variant('[]'), parse_to_variant('{}')), + (4, parse_to_variant('null'), parse_to_variant('null')), + (5, CAST(NULL AS VARIANT), CAST(NULL AS VARIANT)) + """ + + // Typed scalar values exercise every V2 primitive family used by Doris. + sql """ + INSERT INTO t_variant_basic VALUES + (10, CAST(TRUE AS VARIANT), CAST(FALSE AS VARIANT)), + (11, CAST(CAST(-128 AS TINYINT) AS VARIANT), + CAST(CAST(32767 AS SMALLINT) AS VARIANT)), + (12, CAST(CAST(-2147483648 AS INT) AS VARIANT), + CAST(CAST(9223372036854775807 AS BIGINT) AS VARIANT)), + (13, CAST(CAST(1.25 AS FLOAT) AS VARIANT), + CAST(CAST(-2.5 AS DOUBLE) AS VARIANT)), + (14, CAST(CAST(123456.789 AS DECIMAL(12, 3)) AS VARIANT), + CAST(CAST(-0.000001 AS DECIMAL(18, 6)) AS VARIANT)), + (15, CAST(CAST('plain-string' AS VARCHAR(32)) AS VARIANT), + CAST(CAST('δΈζπ' AS STRING) AS VARIANT)), + (16, CAST(DATE '2024-02-29' AS VARIANT), + CAST(CAST('2024-02-29 12:34:56.123456' AS DATETIMEV2(6)) AS VARIANT)), + (17, CAST(REPEAT('long-value-', 4096) AS VARIANT), + parse_to_variant('{"batch":"large-string"}')) + """ + + def objectRows = spark_paimon """ + SELECT + variant_get(payload, '${root}.object.name', 'string'), + variant_get(payload, '${root}["object"]["address"].city', 'string'), + variant_get(payload, '${root}.array[0]', 'int'), + variant_get(payload, '${root}.array[1]', 'boolean'), + variant_get(payload, '${root}.emptyObject', 'string'), + variant_get(payload, '${root}.emptyArray', 'string'), + variant_get(payload, '${root}.unicode', 'string'), + variant_get(secondary, '${root}.second', 'int') + FROM paimon.${dbName}.t_variant_basic + WHERE id = 1 + """ + assertEquals([ + "doris", "Hangzhou", "1", "true", "{}", "[]", "δΈζπ", "2" + ], objectRows[0].collect { value -> value.toString() }) + + def nullRows = spark_paimon """ + SELECT id, payload IS NULL, + variant_get(payload, '${root}.missing', 'string') IS NULL + FROM paimon.${dbName}.t_variant_basic + WHERE id IN (4, 5) + ORDER BY id + """ + assertEquals(["4", "false", "true"], + nullRows[0].collect { value -> value.toString() }) + assertEquals(["5", "true", "true"], + nullRows[1].collect { value -> value.toString() }) + + def scalarRows = spark_paimon """ Review Comment: This query verifies only a subset of the typed scalar rows inserted above. The TINYINT/SMALLINT pair, minimum INT, FLOAT, the second DECIMAL, the first VARCHAR, and DATETIMEV2 are never read back and checked; the final row-count assertion cannot detect type or value corruption in them. Please assert every inserted primitive family, including both boundary values where a row intentionally carries two different types. That is especially important because this PR changes the binary transport boundary rather than only the SQL surface. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
