This is an automated email from the ASF dual-hosted git repository.
xiong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/main by this push:
new c9724b6a80 [CALCITE-6712] FROM_BASE64 and PARSE_URL's return data type
should always be nullable
c9724b6a80 is described below
commit c9724b6a8058546f2e61203f42377c7ac3f50903
Author: Xiong Duan <[email protected]>
AuthorDate: Sat Nov 30 13:13:38 2024 +0800
[CALCITE-6712] FROM_BASE64 and PARSE_URL's return data type should always
be nullable
---
.../calcite/sql/fun/SqlLibraryOperators.java | 4 +-
.../org/apache/calcite/sql/type/ReturnTypes.java | 14 ++++
site/_docs/reference.md | 4 +-
.../calcite/sql/test/SqlOperatorFixture.java | 12 ++++
.../java/org/apache/calcite/sql/test/SqlTests.java | 9 +++
.../calcite/test/SqlOperatorFixtureImpl.java | 5 ++
.../org/apache/calcite/test/SqlOperatorTest.java | 82 ++++++++++++----------
7 files changed, 87 insertions(+), 43 deletions(-)
diff --git
a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
index 5d349a2158..e8f044f3ac 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
@@ -468,7 +468,7 @@ public abstract class SqlLibraryOperators {
@LibraryOperator(libraries = {HIVE, SPARK})
public static final SqlFunction PARSE_URL =
SqlBasicFunction.create("PARSE_URL",
- ReturnTypes.VARCHAR_NULLABLE,
+ ReturnTypes.VARCHAR_FORCE_NULLABLE,
OperandTypes.STRING_STRING_OPTIONAL_STRING,
SqlFunctionCategory.STRING);
@@ -1836,7 +1836,7 @@ public abstract class SqlLibraryOperators {
@LibraryOperator(libraries = {BIG_QUERY, MYSQL})
public static final SqlFunction FROM_BASE64 =
SqlBasicFunction.create("FROM_BASE64",
- ReturnTypes.VARBINARY_NULLABLE,
+ ReturnTypes.VARBINARY_FORCE_NULLABLE,
OperandTypes.STRING, SqlFunctionCategory.STRING);
@LibraryOperator(libraries = {MYSQL})
diff --git a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java
b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java
index d54f3d977e..12849a60db 100644
--- a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java
+++ b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java
@@ -539,6 +539,13 @@ public abstract class ReturnTypes {
public static final SqlReturnTypeInference VARCHAR_NULLABLE =
VARCHAR.andThen(SqlTypeTransforms.TO_NULLABLE);
+ /**
+ * Type-inference strategy whereby the result type of a call is a nullable
+ * VARCHAR.
+ */
+ public static final SqlReturnTypeInference VARCHAR_FORCE_NULLABLE =
+ VARCHAR.andThen(SqlTypeTransforms.FORCE_NULLABLE);
+
/**
* Type-inference strategy that always returns "VARBINARY".
*/
@@ -552,6 +559,13 @@ public abstract class ReturnTypes {
public static final SqlReturnTypeInference VARBINARY_NULLABLE =
VARBINARY.andThen(SqlTypeTransforms.TO_NULLABLE);
+ /**
+ * Type-inference strategy whereby the result type of a call is a nullable
+ * VARBINARY.
+ */
+ public static final SqlReturnTypeInference VARBINARY_FORCE_NULLABLE =
+ VARBINARY.andThen(SqlTypeTransforms.FORCE_NULLABLE);
+
/**
* Type-inference strategy for Histogram agg support.
*/
diff --git a/site/_docs/reference.md b/site/_docs/reference.md
index d2aa0b8799..4de399ba95 100644
--- a/site/_docs/reference.md
+++ b/site/_docs/reference.md
@@ -2861,7 +2861,7 @@ In the following:
| b | TO_BASE32(string) | Converts the *string*
to base-32 encoded form and returns an encoded string
| b | FROM_BASE32(string) | Returns the decoded
result of a base-32 *string* as a string
| m | TO_BASE64(string) | Converts the *string*
to base-64 encoded form and returns a encoded string
-| b m | FROM_BASE64(string) | Returns the decoded
result of a base-64 *string* as a string
+| b m | FROM_BASE64(string) | Returns the decoded
result of a base-64 *string* as a string. If the input argument is an invalid
base-64 *string* the function returns `NULL`
| b | TO_HEX(binary) | Converts *binary* into
a hexadecimal varchar
| b | FROM_HEX(varchar) | Converts a
hexadecimal-encoded *varchar* into bytes
| b o p r s | LTRIM(string) | Returns *string* with
all blanks removed from the start
@@ -2886,7 +2886,7 @@ In the following:
| b | PARSE_DATETIME(format, string) | Uses format specified
by *format* to convert *string* representation of datetime to a TIMESTAMP value
| b | PARSE_TIME(format, string) | Uses format specified
by *format* to convert *string* representation of time to a TIME value
| b | PARSE_TIMESTAMP(format, string[, timeZone]) | Uses format specified
by *format* to convert *string* representation of timestamp to a TIMESTAMP WITH
LOCAL TIME ZONE value in *timeZone*
-| h s | PARSE_URL(urlString, partToExtract [, keyToExtract] ) | Returns the
specified *partToExtract* from the *urlString*. Valid values for
*partToExtract* include HOST, PATH, QUERY, REF, PROTOCOL, AUTHORITY, FILE, and
USERINFO. *keyToExtract* specifies which query to extract
+| h s | PARSE_URL(urlString, partToExtract [, keyToExtract] ) | Returns the
specified *partToExtract* from the *urlString*. Valid values for
*partToExtract* include HOST, PATH, QUERY, REF, PROTOCOL, AUTHORITY, FILE, and
USERINFO. *keyToExtract* specifies which query to extract. If the first
argument is an invalid url *string* the function returns `NULL`
| b s | POW(numeric1, numeric2) | Returns *numeric1*
raised to the power *numeric2*
| b c h q m o f s p r | POWER(numeric1, numeric2) | Returns *numeric1* raised
to the power of *numeric2*
| p r | RANDOM() | Generates a random
double between 0 and 1 inclusive
diff --git
a/testkit/src/main/java/org/apache/calcite/sql/test/SqlOperatorFixture.java
b/testkit/src/main/java/org/apache/calcite/sql/test/SqlOperatorFixture.java
index 94436699b2..3c3a825af6 100644
--- a/testkit/src/main/java/org/apache/calcite/sql/test/SqlOperatorFixture.java
+++ b/testkit/src/main/java/org/apache/calcite/sql/test/SqlOperatorFixture.java
@@ -344,6 +344,18 @@ public interface SqlOperatorFixture extends AutoCloseable {
*/
void checkNull(String expression);
+ /**
+ * Tests that a SQL expression returns the SQL NULL value and
+ * the data type of the returned result is nullable. For example,
+ *
+ * <blockquote>
+ * <pre>checkNullValueWithNullableType("CHAR_LENGTH(CAST(NULL AS
VARCHAR(3))");</pre>
+ * </blockquote>
+ *
+ * @param expression Scalar expression
+ */
+ void checkNullValueWithNullableType(String expression);
+
/**
* Tests that a SQL expression has a given type. For example,
*
diff --git a/testkit/src/main/java/org/apache/calcite/sql/test/SqlTests.java
b/testkit/src/main/java/org/apache/calcite/sql/test/SqlTests.java
index 68b20f1230..7831767acd 100644
--- a/testkit/src/main/java/org/apache/calcite/sql/test/SqlTests.java
+++ b/testkit/src/main/java/org/apache/calcite/sql/test/SqlTests.java
@@ -65,6 +65,15 @@ public abstract class SqlTests {
public static final TypeChecker ANY_TYPE_CHECKER = (sql, type) -> {
};
+ /**
+ * Checker which enforces that a data type is nullable.
+ */
+ public static final TypeChecker ANY_NULLABLE_TYPE_CHECKER = (sql, type) -> {
+ if (!type.isNullable()) {
+ fail("expected nullable, got " + sql);
+ }
+ };
+
/**
* Checker that allows any number or type of parameters.
*/
diff --git
a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorFixtureImpl.java
b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorFixtureImpl.java
index c274875674..8663ebc0aa 100644
--- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorFixtureImpl.java
+++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorFixtureImpl.java
@@ -288,4 +288,9 @@ class SqlOperatorFixtureImpl implements SqlOperatorFixture {
tester.forEachQuery(factory, expression, sql ->
tester.check(factory, sql, SqlTests.ANY_TYPE_CHECKER, isNullValue()));
}
+
+ @Override public void checkNullValueWithNullableType(String expression) {
+ tester.forEachQuery(factory, expression, sql ->
+ tester.check(factory, sql, SqlTests.ANY_NULLABLE_TYPE_CHECKER,
isNullValue()));
+ }
}
diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
index 13e96d4bc0..aacc5f8955 100644
--- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
+++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
@@ -4796,82 +4796,86 @@ public class SqlOperatorTest {
f.checkString("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
+ " 'HOST')",
"calcite.apache.org",
- "VARCHAR NOT NULL");
+ "VARCHAR");
f.checkString("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
+ " 'PATH')",
"/path1/p.php",
- "VARCHAR NOT NULL");
+ "VARCHAR");
f.checkString("parse_url('http://calcite.apache.org/path1/%20p.php?k1=v1&k2=v2#Ref1',"
+ " 'PATH')",
"/path1/%20p.php",
- "VARCHAR NOT NULL");
+ "VARCHAR");
f.checkString("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
+ " 'QUERY')",
"k1=v1&k2=v2",
- "VARCHAR NOT NULL");
+ "VARCHAR");
f.checkString("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
+ " 'REF')",
"Ref1",
- "VARCHAR NOT NULL");
+ "VARCHAR");
f.checkString("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
+ " 'QUERY', 'k2')",
"v2",
- "VARCHAR NOT NULL");
+ "VARCHAR");
f.checkString("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
+ " 'QUERY', 'k1')",
"v1",
- "VARCHAR NOT NULL");
-
f.checkNull("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
- + " 'QUERY', 'k3')");
+ "VARCHAR");
+ f.checkNullValueWithNullableType(
+ "parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',
'QUERY', 'k3')");
f.checkString("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
+ " 'FILE')",
"/path1/p.php?k1=v1&k2=v2",
- "VARCHAR NOT NULL");
+ "VARCHAR");
f.checkString("parse_url('http://calcite.apache.org/path1/p.php',"
+ " 'FILE')",
"/path1/p.php",
- "VARCHAR NOT NULL");
+ "VARCHAR");
f.checkString("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
+ " 'PROTOCOL')",
"http",
- "VARCHAR NOT NULL");
+ "VARCHAR");
f.checkString("parse_url('https://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
+ " 'PROTOCOL')",
"https",
- "VARCHAR NOT NULL");
+ "VARCHAR");
f.checkString("parse_url('http://[email protected]/path1/p.php?k1=v1&k2=v2#Ref1',"
+ " 'USERINFO')",
"bob",
- "VARCHAR NOT NULL");
-
f.checkNull("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
- + " 'USERINFO')");
+ "VARCHAR");
+ f.checkNullValueWithNullableType(
+ "parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',
'USERINFO')");
f.checkString("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
+ " 'AUTHORITY')",
"calcite.apache.org",
- "VARCHAR NOT NULL");
+ "VARCHAR");
// test with invalid partToExtract
-
f.checkNull("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
- + " 'INVALID_PART_TO_EXTRACT')");
-
f.checkNull("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
- + " 'HOST', 'k1')");
+ f.checkNullValueWithNullableType(
+ "parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
+ + " 'INVALID_PART_TO_EXTRACT')");
+ f.checkNullValueWithNullableType(
+ "parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',
'HOST', 'k1')");
// test with invalid urlString
-
f.checkNull("parse_url('http:calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
- + " 'HOST')");
-
f.checkNull("parse_url('calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
- + " 'HOST')");
- f.checkNull("parse_url('/path1/p.php?k1=v1&k2=v2#Ref1',"
- + " 'HOST')");
+ f.checkNullValueWithNullableType(
+ "parse_url('http:calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',
'HOST')");
+ f.checkNullValueWithNullableType(
+ "parse_url('calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',
'HOST')");
+ f.checkNullValueWithNullableType(
+ "parse_url('/path1/p.php?k1=v1&k2=v2#Ref1', 'HOST')");
// test with operands with null values
-
f.checkNull("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
- + " cast(null as varchar))");
-
f.checkNull("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
+ f.checkNullValueWithNullableType(
+ "parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
+ + " cast(null as varchar))");
+ f.checkNullValueWithNullableType(
+ "parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1',"
+ " cast(null as varchar), cast(null as varchar))");
- f.checkNull("parse_url(cast(null as varchar), cast(null as varchar))");
- f.checkNull("parse_url(cast(null as varchar), cast(null as varchar),"
- + " cast(null as varchar))");
+ f.checkNullValueWithNullableType(
+ "parse_url(cast(null as varchar), cast(null as varchar))");
+ f.checkNullValueWithNullableType(
+ "parse_url(cast(null as varchar), cast(null as varchar), cast(null
as varchar))");
};
f0.forEachLibrary(list(SqlLibrary.HIVE, SqlLibrary.SPARK), consumer);
}
@@ -5423,18 +5427,18 @@ public class SqlOperatorTest {
final Consumer<SqlOperatorFixture> consumer = f -> {
f.checkString("from_base64('VGhpcyBpcyBhIHRlc3QgU3RyaW5nLg==')",
"546869732069732061207465737420537472696e672e",
- "VARBINARY NOT NULL");
+ "VARBINARY");
f.checkString("from_base64('VGhpcyBpcyBhIHRlc\t3QgU3RyaW5nLg==')",
"546869732069732061207465737420537472696e672e",
- "VARBINARY NOT NULL");
+ "VARBINARY");
f.checkString("from_base64('VGhpcyBpcyBhIHRlc\t3QgU3\nRyaW5nLg==')",
"546869732069732061207465737420537472696e672e",
- "VARBINARY NOT NULL");
+ "VARBINARY");
f.checkString("from_base64('VGhpcyB pcyBhIHRlc3Qg\tU3Ry\naW5nLg==')",
"546869732069732061207465737420537472696e672e",
- "VARBINARY NOT NULL");
- f.checkNull("from_base64('-1')");
- f.checkNull("from_base64('-100')");
+ "VARBINARY");
+ f.checkNullValueWithNullableType("from_base64('-1')");
+ f.checkNullValueWithNullableType("from_base64('-100')");
};
f0.forEachLibrary(list(SqlLibrary.BIG_QUERY, SqlLibrary.MYSQL), consumer);
}
