This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 8faa524045ea [SPARK-58307][SQL][PYTHON] Add json_typeof function
8faa524045ea is described below
commit 8faa524045eac27e96bf6b67420edf907890821d
Author: SreeramaYeshwanthGowd <[email protected]>
AuthorDate: Sun Aug 9 11:07:06 2026 +0800
[SPARK-58307][SQL][PYTHON] Add json_typeof function
### What changes were proposed in this pull request?
Add a built in `json_typeof(json)` scalar function that returns the type of
the outermost value in a JSON string, as one of `object`, `array`, `string`,
`number`, `boolean`, or `null`. It returns SQL `NULL` when the input is not a
valid JSON string or is an empty string.
API surface added:
- SQL: `json_typeof(json)`
- Scala DataFrame: `functions.json_typeof(col)`
- PySpark, classic and Spark Connect:
`pyspark.sql.functions.json_typeof(col)`
Implementation notes:
- Modeled on the existing `json_object_keys` function: a
`RuntimeReplaceable` expression whose replacement is a `StaticInvoke` into
`JsonExpressionUtils`, using the same Jackson parser (`CreateJacksonParser` /
`SharedFactory`) that already backs `json_array_length` and `json_object_keys`.
No new dependency.
- Like those functions, malformed input returns `NULL`: `json_typeof` reads
the outermost value and consumes it, so a structurally invalid JSON value
returns `NULL` (consistent with `json_object_keys`).
- Both integer and floating point JSON numbers map to `number`, matching
PostgreSQL and BigQuery.
### Why are the changes needed?
Spark can already ask a JSON string for its array length
(`json_array_length`) and its object keys (`json_object_keys`), but not for the
type of its outermost value. `json_typeof` completes that family. It is a
lightweight top level type probe: unlike `schema_of_json`, which does full
recursive schema inference and returns a DDL string such as `ARRAY<BIGINT>`,
`json_typeof` returns a single simple type token. It is provided by PostgreSQL
(`json_typeof` and `jsonb_typeof`) and BigQuery [...]
### Does this PR introduce _any_ user-facing change?
Yes. It adds a new built in SQL function `json_typeof` and the
corresponding Scala and PySpark DataFrame API entries. No existing behavior
changes.
Example:
```
spark-sql> SELECT json_typeof('{"a": 1}');
object
spark-sql> SELECT json_typeof('[1, 2, 3]');
array
spark-sql> SELECT json_typeof('123');
number
spark-sql> SELECT json_typeof('not json');
NULL
```
### How was this patch tested?
Added catalyst unit tests in `JsonExpressionsSuite` covering each JSON type
(object, array, string, number, boolean, null), invalid and empty inputs
returning null, and null propagation, and a DataFrame API test in
`JsonFunctionsSuite`. Added a PySpark doctest and regenerated
`sql-expression-schema.md`.
### Was this patch authored or co-authored using generative AI tooling? No
Closes #57479 from SreeramaYeshwanthGowd/add-json-typeof-function.
Authored-by: SreeramaYeshwanthGowd <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit b24247bac586c8014211fc73f207c5a46af73024)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../source/reference/pyspark.sql/functions.rst | 1 +
python/pyspark/sql/connect/functions/builtin.py | 7 +
python/pyspark/sql/functions/__init__.py | 1 +
python/pyspark/sql/functions/builtin.py | 36 ++++
.../scala/org/apache/spark/sql/functions.scala | 13 ++
.../expressions/json/JsonExpressionUtils.java | 40 +++++
.../sql/catalyst/analysis/FunctionRegistry.scala | 1 +
.../sql/catalyst/expressions/jsonExpressions.scala | 46 +++++
.../expressions/JsonExpressionsSuite.scala | 27 +++
.../apache/spark/sql/PlanGenerationTestSuite.scala | 4 +
.../explain-results/function_json_typeof.explain | 2 +
.../query-tests/queries/function_json_typeof.json | 60 +++++++
.../queries/function_json_typeof.proto.bin | Bin 0 -> 547 bytes
.../sql-functions/sql-expression-schema.md | 1 +
.../analyzer-results/json-functions.sql.out | 166 ++++++++++++++++++
.../resources/sql-tests/inputs/json-functions.sql | 21 +++
.../sql-tests/results/json-functions.sql.out | 187 +++++++++++++++++++++
.../org/apache/spark/sql/JsonFunctionsSuite.scala | 9 +
18 files changed, 622 insertions(+)
diff --git a/python/docs/source/reference/pyspark.sql/functions.rst
b/python/docs/source/reference/pyspark.sql/functions.rst
index cc1756304549..772ffd15bf1a 100644
--- a/python/docs/source/reference/pyspark.sql/functions.rst
+++ b/python/docs/source/reference/pyspark.sql/functions.rst
@@ -589,6 +589,7 @@ JSON Functions
json_array_length
json_object_keys
json_tuple
+ json_typeof
schema_of_json
to_json
diff --git a/python/pyspark/sql/connect/functions/builtin.py
b/python/pyspark/sql/connect/functions/builtin.py
index 0713eecbe9b0..6fc7de84add6 100644
--- a/python/pyspark/sql/connect/functions/builtin.py
+++ b/python/pyspark/sql/connect/functions/builtin.py
@@ -2067,6 +2067,13 @@ def json_object_keys(col: "ColumnOrName") -> Column:
json_object_keys.__doc__ = pysparkfuncs.json_object_keys.__doc__
+def json_typeof(col: "ColumnOrName") -> Column:
+ return _invoke_function_over_columns("json_typeof", col)
+
+
+json_typeof.__doc__ = pysparkfuncs.json_typeof.__doc__
+
+
def inline(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("inline", col)
diff --git a/python/pyspark/sql/functions/__init__.py
b/python/pyspark/sql/functions/__init__.py
index 0e2c6386a4d4..a8212f7372a2 100644
--- a/python/pyspark/sql/functions/__init__.py
+++ b/python/pyspark/sql/functions/__init__.py
@@ -474,6 +474,7 @@ __all__ = [ # noqa: F405
"json_array_length",
"json_object_keys",
"json_tuple",
+ "json_typeof",
"schema_of_json",
"to_json",
# VARIANT Functions
diff --git a/python/pyspark/sql/functions/builtin.py
b/python/pyspark/sql/functions/builtin.py
index 6ed9a1049858..5caf3ae0bf84 100644
--- a/python/pyspark/sql/functions/builtin.py
+++ b/python/pyspark/sql/functions/builtin.py
@@ -23591,6 +23591,42 @@ def json_object_keys(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("json_object_keys", col)
+@_try_remote_functions
+def json_typeof(col: "ColumnOrName") -> Column:
+ """
+ Returns the type of the outermost JSON value as a string: one of 'object',
'array',
+ 'string', 'number', 'boolean', or 'null'. Returns null if the input is not
a valid JSON
+ string or is an empty string.
+
+ .. versionadded:: 4.4.0
+
+ Parameters
+ ----------
+ col: :class:`~pyspark.sql.Column` or str
+ target column to compute on.
+ A column that evaluates to a string.
+
+ Returns
+ -------
+ :class:`~pyspark.sql.Column`
+ the type of the outermost JSON value.
+ Returns a column that evaluates to a string.
+
+ See Also
+ --------
+ :meth:`pyspark.sql.functions.json_object_keys`
+ :meth:`pyspark.sql.functions.get_json_object`
+ :meth:`pyspark.sql.functions.json_array_length`
+
+ Examples
+ --------
+ >>> df = spark.createDataFrame([('{"a": 1}',), ('[1, 2, 3]',), ('123',),
('',)], ['data'])
+ >>> df.select(json_typeof(df.data).alias('r')).collect()
+ [Row(r='object'), Row(r='array'), Row(r='number'), Row(r=None)]
+ """
+ return _invoke_function_over_columns("json_typeof", col)
+
+
# TODO: Fix and add an example for StructType with Spark Connect
# e.g., StructType([StructField("a", IntegerType())])
@_try_remote_functions
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
index 0cc822c2ec01..461eeeef8f6d 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
@@ -14732,6 +14732,19 @@ object functions {
*/
def json_object_keys(e: Column): Column = Column.fn("json_object_keys", e)
+ /**
+ * Returns the type of the outermost JSON value as a string: one of
'object', 'array', 'string',
+ * 'number', 'boolean', or 'null'. Returns null for invalid or empty input.
+ *
+ * @param e
+ * the JSON string column. A column that evaluates to a string.
+ * @group json_funcs
+ * @since 4.4.0
+ * @return
+ * Returns a column that evaluates to a string.
+ */
+ def json_typeof(e: Column): Column = Column.fn("json_typeof", e)
+
// scalastyle:off line.size.limit
/**
* (Scala-specific) Converts a column containing a `StructType`, `ArrayType`
or a `MapType` into
diff --git
a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/json/JsonExpressionUtils.java
b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/json/JsonExpressionUtils.java
index 38bdcbec2069..a3cdc355d75a 100644
---
a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/json/JsonExpressionUtils.java
+++
b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/json/JsonExpressionUtils.java
@@ -77,4 +77,44 @@ public class JsonExpressionUtils {
return null;
}
}
+
+ private static final UTF8String JSON_TYPE_OBJECT =
UTF8String.fromString("object");
+ private static final UTF8String JSON_TYPE_ARRAY =
UTF8String.fromString("array");
+ private static final UTF8String JSON_TYPE_STRING =
UTF8String.fromString("string");
+ private static final UTF8String JSON_TYPE_NUMBER =
UTF8String.fromString("number");
+ private static final UTF8String JSON_TYPE_BOOLEAN =
UTF8String.fromString("boolean");
+ private static final UTF8String JSON_TYPE_NULL =
UTF8String.fromString("null");
+
+ public static UTF8String jsonTypeof(UTF8String json) {
+ try (JsonParser jsonParser =
+ CreateJacksonParser.utf8String(SharedFactory.jsonFactory(), json)) {
+ JsonToken token = jsonParser.nextToken();
+ if (token == null) {
+ return null;
+ }
+ UTF8String type = switch (token) {
+ case START_OBJECT -> JSON_TYPE_OBJECT;
+ case START_ARRAY -> JSON_TYPE_ARRAY;
+ case VALUE_STRING -> JSON_TYPE_STRING;
+ case VALUE_NUMBER_INT, VALUE_NUMBER_FLOAT -> JSON_TYPE_NUMBER;
+ case VALUE_TRUE, VALUE_FALSE -> JSON_TYPE_BOOLEAN;
+ case VALUE_NULL -> JSON_TYPE_NULL;
+ default -> null;
+ };
+ if (type == null) {
+ return null;
+ }
+ // Consume the value so malformed input surfaces as a parse error and
returns null,
+ // matching json_object_keys and json_array_length.
+ jsonParser.skipChildren();
+ // Reject trailing content after the first value, e.g. `123 true`, so
only a single
+ // well-formed JSON value is accepted.
+ if (jsonParser.nextToken() != null) {
+ return null;
+ }
+ return type;
+ } catch (IOException e) {
+ return null;
+ }
+ }
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index e80544a6e260..8f86528bcac0 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -978,6 +978,7 @@ object FunctionRegistry {
expression[SchemaOfJson]("schema_of_json"),
expression[LengthOfJsonArray]("json_array_length"),
expression[JsonObjectKeys]("json_object_keys"),
+ expression[JsonTypeof]("json_typeof"),
// Variant
expressionBuilder("parse_json", ParseJsonExpressionBuilder),
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
index f01d34777bac..577956bb15bd 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
@@ -1022,3 +1022,49 @@ case class JsonObjectKeys(child: Expression)
override protected def withNewChildInternal(newChild: Expression):
JsonObjectKeys =
copy(child = newChild)
}
+
+/**
+ * A function which returns the type of the outermost JSON value as a string.
+ */
+@ExpressionDescription(
+ usage = "_FUNC_(json) - Returns the type of the outermost JSON value, or
null if invalid.",
+ arguments = """
+ Arguments:
+ * json - A JSON string. Returns the type of the outermost value
('object', 'array',
+ 'string', 'number', 'boolean', 'null'), or null for an invalid or
empty string.
+ An expression that evaluates to a string.
+ """,
+ examples = """
+ Examples:
+ > SELECT _FUNC_('{"a": 1}');
+ object
+ > SELECT _FUNC_('[1, 2, 3]');
+ array
+ > SELECT _FUNC_('123');
+ number
+ """,
+ group = "json_funcs",
+ since = "4.4.0"
+)
+case class JsonTypeof(child: Expression)
+ extends UnaryExpression
+ with ExpectsInputTypes
+ with RuntimeReplaceable
+ with DefaultStringProducingExpression {
+
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeWithCollation(supportsTrimCollation = true))
+ override def nullable: Boolean = true
+ override def prettyName: String = "json_typeof"
+
+ override def replacement: Expression = StaticInvoke(
+ classOf[JsonExpressionUtils],
+ dataType,
+ "jsonTypeof",
+ Seq(child),
+ inputTypes
+ )
+
+ override protected def withNewChildInternal(newChild: Expression):
JsonTypeof =
+ copy(child = newChild)
+}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
index 160470b72266..fb8641659a74 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
@@ -909,6 +909,33 @@ class JsonExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
}
}
+ test("json_typeof") {
+ Seq(
+ // Invalid or empty inputs return null.
+ ("", null),
+ ("bad", null),
+ ("""{"key": 45, "random_string"}""", null),
+ // Trailing content after a valid value is not a single well-formed JSON
document.
+ ("123 true", null),
+ // Valid JSON values return the type of the outermost value.
+ ("{}", "object"),
+ ("""{"key": 1, "arr": [1, 2]}""", "object"),
+ ("[]", "array"),
+ ("[1, 2, 3]", "array"),
+ ("\"hello\"", "string"),
+ ("123", "number"),
+ ("1.5", "number"),
+ ("-123", "number"),
+ ("-1.5", "number"),
+ ("true", "boolean"),
+ ("false", "boolean"),
+ ("null", "null")
+ ).foreach {
+ case (input, expected) =>
+ checkEvaluation(JsonTypeof(Literal(input)), expected)
+ }
+ }
+
test("SPARK-35320: from_json should fail with a key type different of
StringType") {
Seq(
(MapType(IntegerType, StringType), """{"1": "test"}"""),
diff --git
a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
index 446efbf0c955..a8e3f2d3e84e 100644
---
a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
+++
b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
@@ -3137,6 +3137,10 @@ class PlanGenerationTestSuite extends ConnectFunSuite
with Logging {
fn.json_object_keys(fn.col("g"))
}
+ functionTest("json_typeof") {
+ fn.json_typeof(fn.col("g"))
+ }
+
functionTest("mask with specific upperChar lowerChar digitChar otherChar") {
fn.mask(fn.col("g"), fn.lit('X'), fn.lit('x'), fn.lit('n'), fn.lit('*'))
}
diff --git
a/sql/connect/common/src/test/resources/query-tests/explain-results/function_json_typeof.explain
b/sql/connect/common/src/test/resources/query-tests/explain-results/function_json_typeof.explain
new file mode 100644
index 000000000000..c8d3652edcb1
--- /dev/null
+++
b/sql/connect/common/src/test/resources/query-tests/explain-results/function_json_typeof.explain
@@ -0,0 +1,2 @@
+Project [static_invoke(JsonExpressionUtils.jsonTypeof(g#0)) AS
json_typeof(g)#0]
++- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git
a/sql/connect/common/src/test/resources/query-tests/queries/function_json_typeof.json
b/sql/connect/common/src/test/resources/query-tests/queries/function_json_typeof.json
new file mode 100644
index 000000000000..a2f300b98ee8
--- /dev/null
+++
b/sql/connect/common/src/test/resources/query-tests/queries/function_json_typeof.json
@@ -0,0 +1,60 @@
+{
+ "common": {
+ "planId": "1"
+ },
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema":
"struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+ }
+ },
+ "expressions": [{
+ "unresolvedFunction": {
+ "functionName": "json_typeof",
+ "arguments": [{
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "g"
+ },
+ "common": {
+ "origin": {
+ "jvmOrigin": {
+ "stackTrace": [{
+ "classLoaderName": "app",
+ "declaringClass": "org.apache.spark.sql.functions$",
+ "methodName": "col",
+ "fileName": "functions.scala"
+ }, {
+ "classLoaderName": "app",
+ "declaringClass":
"org.apache.spark.sql.PlanGenerationTestSuite",
+ "methodName": "~~trimmed~anonfun~~",
+ "fileName": "PlanGenerationTestSuite.scala"
+ }]
+ }
+ }
+ }
+ }],
+ "isInternal": false
+ },
+ "common": {
+ "origin": {
+ "jvmOrigin": {
+ "stackTrace": [{
+ "classLoaderName": "app",
+ "declaringClass": "org.apache.spark.sql.functions$",
+ "methodName": "json_typeof",
+ "fileName": "functions.scala"
+ }, {
+ "classLoaderName": "app",
+ "declaringClass": "org.apache.spark.sql.PlanGenerationTestSuite",
+ "methodName": "~~trimmed~anonfun~~",
+ "fileName": "PlanGenerationTestSuite.scala"
+ }]
+ }
+ }
+ }
+ }]
+ }
+}
\ No newline at end of file
diff --git
a/sql/connect/common/src/test/resources/query-tests/queries/function_json_typeof.proto.bin
b/sql/connect/common/src/test/resources/query-tests/queries/function_json_typeof.proto.bin
new file mode 100644
index 000000000000..7177b03938e2
Binary files /dev/null and
b/sql/connect/common/src/test/resources/query-tests/queries/function_json_typeof.proto.bin
differ
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index 10f68f708b1b..7a4c8ffcfbcd 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -188,6 +188,7 @@
| org.apache.spark.sql.catalyst.expressions.JsonObjectKeys | json_object_keys
| SELECT json_object_keys('{}') | struct<json_object_keys({}):array<string>> |
| org.apache.spark.sql.catalyst.expressions.JsonToStructs | from_json | SELECT
from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE') | struct<from_json({"a":1,
"b":0.8}):struct<a:int,b:double>> |
| org.apache.spark.sql.catalyst.expressions.JsonTuple | json_tuple | SELECT
json_tuple('{"a":1, "b":2}', 'a', 'b') | struct<c0:string,c1:string> |
+| org.apache.spark.sql.catalyst.expressions.JsonTypeof | json_typeof | SELECT
json_typeof('{"a": 1}') | struct<json_typeof({"a": 1}):string> |
| org.apache.spark.sql.catalyst.expressions.KllSketchGetNBigint |
kll_sketch_get_n_bigint | SELECT
kll_sketch_get_n_bigint(kll_sketch_agg_bigint(col)) FROM VALUES (1), (2), (3),
(4), (5) tab(col) |
struct<kll_sketch_get_n_bigint(kll_sketch_agg_bigint(col)):bigint> |
| org.apache.spark.sql.catalyst.expressions.KllSketchGetNDouble |
kll_sketch_get_n_double | SELECT
kll_sketch_get_n_double(kll_sketch_agg_double(col)) FROM VALUES (CAST(1.0 AS
DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)),
(CAST(5.0 AS DOUBLE)) tab(col) |
struct<kll_sketch_get_n_double(kll_sketch_agg_double(col)):bigint> |
| org.apache.spark.sql.catalyst.expressions.KllSketchGetNFloat |
kll_sketch_get_n_float | SELECT
kll_sketch_get_n_float(kll_sketch_agg_float(col)) FROM VALUES (CAST(1.0 AS
FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)),
(CAST(5.0 AS FLOAT)) tab(col) |
struct<kll_sketch_get_n_float(kll_sketch_agg_float(col)):bigint> |
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/json-functions.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/json-functions.sql.out
index 4fb1f0f04231..0582c531d570 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/json-functions.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/json-functions.sql.out
@@ -742,6 +742,172 @@ Project [json_object_keys([1, 2, 3]) AS
json_object_keys([1, 2, 3])#x]
+- OneRowRelation
+-- !query
+select json_typeof()
+-- !query analysis
+org.apache.spark.sql.AnalysisException
+{
+ "errorClass" : "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ "sqlState" : "42605",
+ "messageParameters" : {
+ "actualNum" : "0",
+ "docroot" : "https://spark.apache.org/docs/latest";,
+ "expectedNum" : "1",
+ "functionName" : "`json_typeof`"
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 20,
+ "fragment" : "json_typeof()"
+ } ]
+}
+
+
+-- !query
+select json_typeof(null)
+-- !query analysis
+Project [json_typeof(null) AS json_typeof(NULL)#x]
++- OneRowRelation
+
+
+-- !query
+select json_typeof(200)
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ "sqlState" : "42K09",
+ "messageParameters" : {
+ "inputSql" : "\"200\"",
+ "inputType" : "\"INT\"",
+ "paramIndex" : "first",
+ "requiredType" : "\"STRING\"",
+ "sqlExpr" : "\"json_typeof(200)\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 23,
+ "fragment" : "json_typeof(200)"
+ } ]
+}
+
+
+-- !query
+select json_typeof('')
+-- !query analysis
+Project [json_typeof() AS json_typeof()#x]
++- OneRowRelation
+
+
+-- !query
+select json_typeof('{}')
+-- !query analysis
+Project [json_typeof({}) AS json_typeof({})#x]
++- OneRowRelation
+
+
+-- !query
+select json_typeof('{"key": 1, "arr": [1, 2]}')
+-- !query analysis
+Project [json_typeof({"key": 1, "arr": [1, 2]}) AS json_typeof({"key": 1,
"arr": [1, 2]})#x]
++- OneRowRelation
+
+
+-- !query
+select json_typeof('[]')
+-- !query analysis
+Project [json_typeof([]) AS json_typeof([])#x]
++- OneRowRelation
+
+
+-- !query
+select json_typeof('[1, 2, 3]')
+-- !query analysis
+Project [json_typeof([1, 2, 3]) AS json_typeof([1, 2, 3])#x]
++- OneRowRelation
+
+
+-- !query
+select json_typeof('"hello"')
+-- !query analysis
+Project [json_typeof("hello") AS json_typeof("hello")#x]
++- OneRowRelation
+
+
+-- !query
+select json_typeof('123')
+-- !query analysis
+Project [json_typeof(123) AS json_typeof(123)#x]
++- OneRowRelation
+
+
+-- !query
+select json_typeof('1.5')
+-- !query analysis
+Project [json_typeof(1.5) AS json_typeof(1.5)#x]
++- OneRowRelation
+
+
+-- !query
+select json_typeof('-123')
+-- !query analysis
+Project [json_typeof(-123) AS json_typeof(-123)#x]
++- OneRowRelation
+
+
+-- !query
+select json_typeof('-1.5')
+-- !query analysis
+Project [json_typeof(-1.5) AS json_typeof(-1.5)#x]
++- OneRowRelation
+
+
+-- !query
+select json_typeof('true')
+-- !query analysis
+Project [json_typeof(true) AS json_typeof(true)#x]
++- OneRowRelation
+
+
+-- !query
+select json_typeof('false')
+-- !query analysis
+Project [json_typeof(false) AS json_typeof(false)#x]
++- OneRowRelation
+
+
+-- !query
+select json_typeof('null')
+-- !query analysis
+Project [json_typeof(null) AS json_typeof(null)#x]
++- OneRowRelation
+
+
+-- !query
+select json_typeof('bad')
+-- !query analysis
+Project [json_typeof(bad) AS json_typeof(bad)#x]
++- OneRowRelation
+
+
+-- !query
+select json_typeof('{"key": 45, "random_string"}')
+-- !query analysis
+Project [json_typeof({"key": 45, "random_string"}) AS json_typeof({"key": 45,
"random_string"})#x]
++- OneRowRelation
+
+
+-- !query
+select json_typeof('123 true')
+-- !query analysis
+Project [json_typeof(123 true) AS json_typeof(123 true)#x]
++- OneRowRelation
+
+
-- !query
DROP VIEW IF EXISTS jsonTable
-- !query analysis
diff --git a/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
b/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
index 66134545107a..2a1fae4a6b45 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
@@ -108,6 +108,27 @@ select json_object_keys('{[1,2]}');
select json_object_keys('{"key": 45, "random_string"}');
select json_object_keys('[1, 2, 3]');
+-- json_typeof
+select json_typeof();
+select json_typeof(null);
+select json_typeof(200);
+select json_typeof('');
+select json_typeof('{}');
+select json_typeof('{"key": 1, "arr": [1, 2]}');
+select json_typeof('[]');
+select json_typeof('[1, 2, 3]');
+select json_typeof('"hello"');
+select json_typeof('123');
+select json_typeof('1.5');
+select json_typeof('-123');
+select json_typeof('-1.5');
+select json_typeof('true');
+select json_typeof('false');
+select json_typeof('null');
+select json_typeof('bad');
+select json_typeof('{"key": 45, "random_string"}');
+select json_typeof('123 true');
+
-- Clean up
DROP VIEW IF EXISTS jsonTable;
diff --git
a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
index 96c6dab19dc7..adfe51d05bca 100644
--- a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
@@ -825,6 +825,193 @@ struct<json_object_keys([1, 2, 3]):array<string>>
NULL
+-- !query
+select json_typeof()
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+{
+ "errorClass" : "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ "sqlState" : "42605",
+ "messageParameters" : {
+ "actualNum" : "0",
+ "docroot" : "https://spark.apache.org/docs/latest";,
+ "expectedNum" : "1",
+ "functionName" : "`json_typeof`"
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 20,
+ "fragment" : "json_typeof()"
+ } ]
+}
+
+
+-- !query
+select json_typeof(null)
+-- !query schema
+struct<json_typeof(NULL):string>
+-- !query output
+NULL
+
+
+-- !query
+select json_typeof(200)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ "sqlState" : "42K09",
+ "messageParameters" : {
+ "inputSql" : "\"200\"",
+ "inputType" : "\"INT\"",
+ "paramIndex" : "first",
+ "requiredType" : "\"STRING\"",
+ "sqlExpr" : "\"json_typeof(200)\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 23,
+ "fragment" : "json_typeof(200)"
+ } ]
+}
+
+
+-- !query
+select json_typeof('')
+-- !query schema
+struct<json_typeof():string>
+-- !query output
+NULL
+
+
+-- !query
+select json_typeof('{}')
+-- !query schema
+struct<json_typeof({}):string>
+-- !query output
+object
+
+
+-- !query
+select json_typeof('{"key": 1, "arr": [1, 2]}')
+-- !query schema
+struct<json_typeof({"key": 1, "arr": [1, 2]}):string>
+-- !query output
+object
+
+
+-- !query
+select json_typeof('[]')
+-- !query schema
+struct<json_typeof([]):string>
+-- !query output
+array
+
+
+-- !query
+select json_typeof('[1, 2, 3]')
+-- !query schema
+struct<json_typeof([1, 2, 3]):string>
+-- !query output
+array
+
+
+-- !query
+select json_typeof('"hello"')
+-- !query schema
+struct<json_typeof("hello"):string>
+-- !query output
+string
+
+
+-- !query
+select json_typeof('123')
+-- !query schema
+struct<json_typeof(123):string>
+-- !query output
+number
+
+
+-- !query
+select json_typeof('1.5')
+-- !query schema
+struct<json_typeof(1.5):string>
+-- !query output
+number
+
+
+-- !query
+select json_typeof('-123')
+-- !query schema
+struct<json_typeof(-123):string>
+-- !query output
+number
+
+
+-- !query
+select json_typeof('-1.5')
+-- !query schema
+struct<json_typeof(-1.5):string>
+-- !query output
+number
+
+
+-- !query
+select json_typeof('true')
+-- !query schema
+struct<json_typeof(true):string>
+-- !query output
+boolean
+
+
+-- !query
+select json_typeof('false')
+-- !query schema
+struct<json_typeof(false):string>
+-- !query output
+boolean
+
+
+-- !query
+select json_typeof('null')
+-- !query schema
+struct<json_typeof(null):string>
+-- !query output
+null
+
+
+-- !query
+select json_typeof('bad')
+-- !query schema
+struct<json_typeof(bad):string>
+-- !query output
+NULL
+
+
+-- !query
+select json_typeof('{"key": 45, "random_string"}')
+-- !query schema
+struct<json_typeof({"key": 45, "random_string"}):string>
+-- !query output
+NULL
+
+
+-- !query
+select json_typeof('123 true')
+-- !query schema
+struct<json_typeof(123 true):string>
+-- !query output
+NULL
+
+
-- !query
DROP VIEW IF EXISTS jsonTable
-- !query schema
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
index 6eccbb7a6a38..2bf18ba469dc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
@@ -1976,6 +1976,15 @@ class JsonFunctionsSuite extends SharedSparkSession {
checkAnswer(df.select(json_object_keys($"a")), expected)
}
+ test("json_typeof function") {
+ val df = Seq(null, "{}", "[1, 2, 3]", "\"str\"", "123", "true", "null",
"", "bad")
+ .toDF("a")
+ val expected = Seq(Row(null), Row("object"), Row("array"), Row("string"),
+ Row("number"), Row("boolean"), Row("null"), Row(null), Row(null))
+ checkAnswer(df.selectExpr("json_typeof(a)"), expected)
+ checkAnswer(df.select(json_typeof($"a")), expected)
+ }
+
test("function get_json_object - Codegen Support") {
withTempView("GetJsonObjectTable") {
val data = Seq(("1", """{"f1": "value1", "f5": 5.23}""")).toDF("key",
"jstring")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
