Repository: spark Updated Branches: refs/heads/master ab5752cb9 -> b8c6ba9e6
[SPARK-25779][SQL][TESTS] Remove SQL query tests for function documentation by DESCRIBE FUNCTION at SQLQueryTestSuite Currently, there are some tests testing function descriptions: ```bash $ grep -ir "describe function" sql/core/src/test/resources/sql-tests/inputs sql/core/src/test/resources/sql-tests/inputs/json-functions.sql:describe function to_json; sql/core/src/test/resources/sql-tests/inputs/json-functions.sql:describe function extended to_json; sql/core/src/test/resources/sql-tests/inputs/json-functions.sql:describe function from_json; sql/core/src/test/resources/sql-tests/inputs/json-functions.sql:describe function extended from_json; ``` Looks there are not quite good points about testing them since we're not going to test documentation itself. For `DESCRIBE FCUNTION` functionality itself, they are already being tested here and there. See the test failures in https://github.com/apache/spark/pull/18749 (where I added examples to function descriptions) We better remove those tests so that people don't add such tests in the SQL tests. ## How was this patch tested? Manual. Closes #22776 from HyukjinKwon/SPARK-25779. Authored-by: hyukjinkwon <gurwls...@apache.org> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b8c6ba9e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b8c6ba9e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b8c6ba9e Branch: refs/heads/master Commit: b8c6ba9e644786e5da7e009eb8030ac375ccd75f Parents: ab5752c Author: hyukjinkwon <gurwls...@apache.org> Authored: Sat Oct 20 18:02:38 2018 -0700 Committer: Dongjoon Hyun <dongj...@apache.org> Committed: Sat Oct 20 18:02:38 2018 -0700 ---------------------------------------------------------------------- .../sql-tests/inputs/json-functions.sql | 4 - .../sql-tests/results/json-functions.sql.out | 296 +++++++------------ 2 files changed, 114 insertions(+), 186 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/b8c6ba9e/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql ---------------------------------------------------------------------- diff --git a/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql index bdd1fe4..8bfd7c0 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql @@ -1,6 +1,4 @@ -- to_json -describe function to_json; -describe function extended to_json; select to_json(named_struct('a', 1, 'b', 2)); select to_json(named_struct('time', to_timestamp('2015-08-26', 'yyyy-MM-dd')), map('timestampFormat', 'dd/MM/yyyy')); select to_json(array(named_struct('a', 1, 'b', 2))); @@ -15,8 +13,6 @@ select to_json(named_struct('a', 1, 'b', 2), map('mode', 1)); select to_json(); -- from_json -describe function from_json; -describe function extended from_json; select from_json('{"a":1}', 'a INT'); select from_json('{"time":"26/08/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy')); -- Check if errors handled http://git-wip-us.apache.org/repos/asf/spark/blob/b8c6ba9e/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out ---------------------------------------------------------------------- diff --git a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out index 868eee8..c70a81e 100644 --- a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out @@ -1,196 +1,128 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 42 +-- Number of queries: 38 -- !query 0 -describe function to_json --- !query 0 schema -struct<function_desc:string> --- !query 0 output -Class: org.apache.spark.sql.catalyst.expressions.StructsToJson -Function: to_json -Usage: to_json(expr[, options]) - Returns a JSON string with a given struct value - - --- !query 1 -describe function extended to_json --- !query 1 schema -struct<function_desc:string> --- !query 1 output -Class: org.apache.spark.sql.catalyst.expressions.StructsToJson -Extended Usage: - Examples: - > SELECT to_json(named_struct('a', 1, 'b', 2)); - {"a":1,"b":2} - > SELECT to_json(named_struct('time', to_timestamp('2015-08-26', 'yyyy-MM-dd')), map('timestampFormat', 'dd/MM/yyyy')); - {"time":"26/08/2015"} - > SELECT to_json(array(named_struct('a', 1, 'b', 2))); - [{"a":1,"b":2}] - > SELECT to_json(map('a', named_struct('b', 1))); - {"a":{"b":1}} - > SELECT to_json(map(named_struct('a', 1),named_struct('b', 2))); - {"[1]":{"b":2}} - > SELECT to_json(map('a', 1)); - {"a":1} - > SELECT to_json(array((map('a', 1)))); - [{"a":1}] - - Since: 2.2.0 - -Function: to_json -Usage: to_json(expr[, options]) - Returns a JSON string with a given struct value - - --- !query 2 select to_json(named_struct('a', 1, 'b', 2)) --- !query 2 schema +-- !query 0 schema struct<to_json(named_struct(a, 1, b, 2)):string> --- !query 2 output +-- !query 0 output {"a":1,"b":2} --- !query 3 +-- !query 1 select to_json(named_struct('time', to_timestamp('2015-08-26', 'yyyy-MM-dd')), map('timestampFormat', 'dd/MM/yyyy')) --- !query 3 schema +-- !query 1 schema struct<to_json(named_struct(time, to_timestamp('2015-08-26', 'yyyy-MM-dd'))):string> --- !query 3 output +-- !query 1 output {"time":"26/08/2015"} --- !query 4 +-- !query 2 select to_json(array(named_struct('a', 1, 'b', 2))) --- !query 4 schema +-- !query 2 schema struct<to_json(array(named_struct(a, 1, b, 2))):string> --- !query 4 output +-- !query 2 output [{"a":1,"b":2}] --- !query 5 +-- !query 3 select to_json(map(named_struct('a', 1, 'b', 2), named_struct('a', 1, 'b', 2))) --- !query 5 schema +-- !query 3 schema struct<to_json(map(named_struct(a, 1, b, 2), named_struct(a, 1, b, 2))):string> --- !query 5 output +-- !query 3 output {"[1,2]":{"a":1,"b":2}} --- !query 6 +-- !query 4 select to_json(map('a', named_struct('a', 1, 'b', 2))) --- !query 6 schema +-- !query 4 schema struct<to_json(map(a, named_struct(a, 1, b, 2))):string> --- !query 6 output +-- !query 4 output {"a":{"a":1,"b":2}} --- !query 7 +-- !query 5 select to_json(map('a', 1)) --- !query 7 schema +-- !query 5 schema struct<to_json(map(a, 1)):string> --- !query 7 output +-- !query 5 output {"a":1} --- !query 8 +-- !query 6 select to_json(array(map('a',1))) --- !query 8 schema +-- !query 6 schema struct<to_json(array(map(a, 1))):string> --- !query 8 output +-- !query 6 output [{"a":1}] --- !query 9 +-- !query 7 select to_json(array(map('a',1), map('b',2))) --- !query 9 schema +-- !query 7 schema struct<to_json(array(map(a, 1), map(b, 2))):string> --- !query 9 output +-- !query 7 output [{"a":1},{"b":2}] --- !query 10 +-- !query 8 select to_json(named_struct('a', 1, 'b', 2), named_struct('mode', 'PERMISSIVE')) --- !query 10 schema +-- !query 8 schema struct<> --- !query 10 output +-- !query 8 output org.apache.spark.sql.AnalysisException Must use a map() function for options;; line 1 pos 7 --- !query 11 +-- !query 9 select to_json(named_struct('a', 1, 'b', 2), map('mode', 1)) --- !query 11 schema +-- !query 9 schema struct<> --- !query 11 output +-- !query 9 output org.apache.spark.sql.AnalysisException A type of keys and values in map() must be string, but got map<string,int>;; line 1 pos 7 --- !query 12 +-- !query 10 select to_json() --- !query 12 schema +-- !query 10 schema struct<> --- !query 12 output +-- !query 10 output org.apache.spark.sql.AnalysisException Invalid number of arguments for function to_json. Expected: one of 1 and 2; Found: 0; line 1 pos 7 --- !query 13 -describe function from_json --- !query 13 schema -struct<function_desc:string> --- !query 13 output -Class: org.apache.spark.sql.catalyst.expressions.JsonToStructs -Function: from_json -Usage: from_json(jsonStr, schema[, options]) - Returns a struct value with the given `jsonStr` and `schema`. - - --- !query 14 -describe function extended from_json --- !query 14 schema -struct<function_desc:string> --- !query 14 output -Class: org.apache.spark.sql.catalyst.expressions.JsonToStructs -Extended Usage: - Examples: - > SELECT from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE'); - {"a":1, "b":0.8} - > SELECT from_json('{"time":"26/08/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy')); - {"time":"2015-08-26 00:00:00.0"} - - Since: 2.2.0 - -Function: from_json -Usage: from_json(jsonStr, schema[, options]) - Returns a struct value with the given `jsonStr` and `schema`. - - --- !query 15 +-- !query 11 select from_json('{"a":1}', 'a INT') --- !query 15 schema +-- !query 11 schema struct<from_json({"a":1}):struct<a:int>> --- !query 15 output +-- !query 11 output {"a":1} --- !query 16 +-- !query 12 select from_json('{"time":"26/08/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy')) --- !query 16 schema +-- !query 12 schema struct<from_json({"time":"26/08/2015"}):struct<time:timestamp>> --- !query 16 output +-- !query 12 output {"time":2015-08-26 00:00:00.0} --- !query 17 +-- !query 13 select from_json('{"a":1}', 1) --- !query 17 schema +-- !query 13 schema struct<> --- !query 17 output +-- !query 13 output org.apache.spark.sql.AnalysisException Schema should be specified in DDL format as a string literal or output of the schema_of_json function instead of 1;; line 1 pos 7 --- !query 18 +-- !query 14 select from_json('{"a":1}', 'a InvalidType') --- !query 18 schema +-- !query 14 schema struct<> --- !query 18 output +-- !query 14 output org.apache.spark.sql.AnalysisException DataType invalidtype is not supported.(line 1, pos 2) @@ -201,188 +133,188 @@ a InvalidType ; line 1 pos 7 --- !query 19 +-- !query 15 select from_json('{"a":1}', 'a INT', named_struct('mode', 'PERMISSIVE')) --- !query 19 schema +-- !query 15 schema struct<> --- !query 19 output +-- !query 15 output org.apache.spark.sql.AnalysisException Must use a map() function for options;; line 1 pos 7 --- !query 20 +-- !query 16 select from_json('{"a":1}', 'a INT', map('mode', 1)) --- !query 20 schema +-- !query 16 schema struct<> --- !query 20 output +-- !query 16 output org.apache.spark.sql.AnalysisException A type of keys and values in map() must be string, but got map<string,int>;; line 1 pos 7 --- !query 21 +-- !query 17 select from_json() --- !query 21 schema +-- !query 17 schema struct<> --- !query 21 output +-- !query 17 output org.apache.spark.sql.AnalysisException Invalid number of arguments for function from_json. Expected: one of 2 and 3; Found: 0; line 1 pos 7 --- !query 22 +-- !query 18 SELECT json_tuple('{"a" : 1, "b" : 2}', CAST(NULL AS STRING), 'b', CAST(NULL AS STRING), 'a') --- !query 22 schema +-- !query 18 schema struct<c0:string,c1:string,c2:string,c3:string> --- !query 22 output +-- !query 18 output NULL 2 NULL 1 --- !query 23 +-- !query 19 CREATE TEMPORARY VIEW jsonTable(jsonField, a) AS SELECT * FROM VALUES ('{"a": 1, "b": 2}', 'a') --- !query 23 schema +-- !query 19 schema struct<> --- !query 23 output +-- !query 19 output --- !query 24 +-- !query 20 SELECT json_tuple(jsonField, 'b', CAST(NULL AS STRING), a) FROM jsonTable --- !query 24 schema +-- !query 20 schema struct<c0:string,c1:string,c2:string> --- !query 24 output +-- !query 20 output 2 NULL 1 --- !query 25 +-- !query 21 DROP VIEW IF EXISTS jsonTable --- !query 25 schema +-- !query 21 schema struct<> --- !query 25 output +-- !query 21 output --- !query 26 +-- !query 22 select from_json('{"a":1, "b":2}', 'map<string, int>') --- !query 26 schema +-- !query 22 schema struct<entries:map<string,int>> --- !query 26 output +-- !query 22 output {"a":1,"b":2} --- !query 27 +-- !query 23 select from_json('{"a":1, "b":"2"}', 'struct<a:int,b:string>') --- !query 27 schema +-- !query 23 schema struct<from_json({"a":1, "b":"2"}):struct<a:int,b:string>> --- !query 27 output +-- !query 23 output {"a":1,"b":"2"} --- !query 28 +-- !query 24 select schema_of_json('{"c1":0, "c2":[1]}') --- !query 28 schema +-- !query 24 schema struct<schema_of_json({"c1":0, "c2":[1]}):string> --- !query 28 output +-- !query 24 output struct<c1:bigint,c2:array<bigint>> --- !query 29 +-- !query 25 select from_json('{"c1":[1, 2, 3]}', schema_of_json('{"c1":[0]}')) --- !query 29 schema +-- !query 25 schema struct<from_json({"c1":[1, 2, 3]}):struct<c1:array<bigint>>> --- !query 29 output +-- !query 25 output {"c1":[1,2,3]} --- !query 30 +-- !query 26 select from_json('[1, 2, 3]', 'array<int>') --- !query 30 schema +-- !query 26 schema struct<from_json([1, 2, 3]):array<int>> --- !query 30 output +-- !query 26 output [1,2,3] --- !query 31 +-- !query 27 select from_json('[1, "2", 3]', 'array<int>') --- !query 31 schema +-- !query 27 schema struct<from_json([1, "2", 3]):array<int>> --- !query 31 output +-- !query 27 output NULL --- !query 32 +-- !query 28 select from_json('[1, 2, null]', 'array<int>') --- !query 32 schema +-- !query 28 schema struct<from_json([1, 2, null]):array<int>> --- !query 32 output +-- !query 28 output [1,2,null] --- !query 33 +-- !query 29 select from_json('[{"a": 1}, {"a":2}]', 'array<struct<a:int>>') --- !query 33 schema +-- !query 29 schema struct<from_json([{"a": 1}, {"a":2}]):array<struct<a:int>>> --- !query 33 output +-- !query 29 output [{"a":1},{"a":2}] --- !query 34 +-- !query 30 select from_json('{"a": 1}', 'array<struct<a:int>>') --- !query 34 schema +-- !query 30 schema struct<from_json({"a": 1}):array<struct<a:int>>> --- !query 34 output +-- !query 30 output [{"a":1}] --- !query 35 +-- !query 31 select from_json('[null, {"a":2}]', 'array<struct<a:int>>') --- !query 35 schema +-- !query 31 schema struct<from_json([null, {"a":2}]):array<struct<a:int>>> --- !query 35 output +-- !query 31 output [null,{"a":2}] --- !query 36 +-- !query 32 select from_json('[{"a": 1}, {"b":2}]', 'array<map<string,int>>') --- !query 36 schema +-- !query 32 schema struct<from_json([{"a": 1}, {"b":2}]):array<map<string,int>>> --- !query 36 output +-- !query 32 output [{"a":1},{"b":2}] --- !query 37 +-- !query 33 select from_json('[{"a": 1}, 2]', 'array<map<string,int>>') --- !query 37 schema +-- !query 33 schema struct<from_json([{"a": 1}, 2]):array<map<string,int>>> --- !query 37 output +-- !query 33 output NULL --- !query 38 +-- !query 34 select to_json(array('1', '2', '3')) --- !query 38 schema +-- !query 34 schema struct<to_json(array(1, 2, 3)):string> --- !query 38 output +-- !query 34 output ["1","2","3"] --- !query 39 +-- !query 35 select to_json(array(array(1, 2, 3), array(4))) --- !query 39 schema +-- !query 35 schema struct<to_json(array(array(1, 2, 3), array(4))):string> --- !query 39 output +-- !query 35 output [[1,2,3],[4]] --- !query 40 +-- !query 36 select schema_of_json('{"c1":1}', map('primitivesAsString', 'true')) --- !query 40 schema +-- !query 36 schema struct<schema_of_json({"c1":1}):string> --- !query 40 output +-- !query 36 output struct<c1:string> --- !query 41 +-- !query 37 select schema_of_json('{"c1":01, "c2":0.1}', map('allowNumericLeadingZeros', 'true', 'prefersDecimal', 'true')) --- !query 41 schema +-- !query 37 schema struct<schema_of_json({"c1":01, "c2":0.1}):string> --- !query 41 output +-- !query 37 output struct<c1:bigint,c2:decimal(1,1)> --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
