This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 149f4fd [FLINK-24803][table-planner] Fix cast BINARY/VARBINARY to
STRING
149f4fd is described below
commit 149f4fd3009641ee081ea5c6c05ddc281e84ba2e
Author: Marios Trivyzas <[email protected]>
AuthorDate: Tue Dec 28 12:23:11 2021 +0200
[FLINK-24803][table-planner] Fix cast BINARY/VARBINARY to STRING
Use an hex string representation when casting any kind of
`BINARY`, `VARBINARY` or `BYTES` to `CHAR`/`VARCHAR`/`STRING`, e.g.:
```
SELECT CAST(CAST(x'68656C6C6F20636F6465' AS BINARY(10)) AS VARCHAR)
```
gives:
```
68656c6c6f20636f6465
```
Apply padding or trimming if needed and also implement, the inverse
cast as well from the hex string to a
`BINARY`/`VARBINARY`/`BYTES` type.
With legacy behaviour enabled we will converting each byte to a UTF8
char and the opposite.
This closes #18221.
---
.../apache/flink/table/utils/EncodingUtils.java | 56 +++++++++
.../functions/casting/ArrayToStringCastRule.java | 2 +-
.../functions/casting/BinaryToStringCastRule.java | 77 ++++++++++++-
.../planner/functions/casting/CastRuleUtils.java | 13 ++-
.../casting/MapAndMultisetToStringCastRule.java | 2 +-
.../functions/casting/StringToBinaryCastRule.java | 29 +++--
.../planner/functions/CastFunctionITCase.java | 32 +++---
.../planner/functions/CastFunctionMiscITCase.java | 4 +-
.../planner/functions/casting/CastRulesTest.java | 125 ++++++++++++---------
.../planner/expressions/ScalarFunctionsTest.scala | 19 +---
10 files changed, 252 insertions(+), 107 deletions(-)
diff --git
a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/utils/EncodingUtils.java
b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/utils/EncodingUtils.java
index c114062..d47779e 100644
---
a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/utils/EncodingUtils.java
+++
b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/utils/EncodingUtils.java
@@ -182,6 +182,62 @@ public abstract class EncodingUtils {
return new String(hexChars);
}
+ /**
+ * Converts an array of characters representing hexadecimal values into an
array of bytes of
+ * those same values. The returned array will be half the length of the
passed array, as it
+ * takes two characters to represent any given byte. An exception is
thrown if the passed char
+ * array has an odd number of elements.
+ *
+ * <p>Copied from
+ *
https://github.com/apache/commons-codec/blob/master/src/main/java/org/apache/commons/codec/binary/Hex.java.
+ *
+ * @param str An array of characters containing hexadecimal digits
+ * @return A byte array to contain the binary data decoded from the
supplied char array.
+ * @throws TableException Thrown if an odd number of characters or illegal
characters are
+ * supplied
+ */
+ public static byte[] decodeHex(final String str) throws TableException {
+ final int len = str.length();
+
+ if ((len & 0x01) != 0) {
+ throw new TableException("Odd number of characters.");
+ }
+
+ final int outLen = len >> 1;
+ final byte[] out = new byte[outLen];
+
+ // two characters form the hex value.
+ for (int i = 0, j = 0; j < len; i++) {
+ int f = toDigit(str.charAt(j), j) << 4;
+ j++;
+ f = f | toDigit(str.charAt(j), j);
+ j++;
+ out[i] = (byte) (f & 0xFF);
+ }
+
+ return out;
+ }
+
+ /**
+ * Converts a hexadecimal character to an integer.
+ *
+ * <p>Copied from
+ *
https://github.com/apache/commons-codec/blob/master/src/main/java/org/apache/commons/codec/binary/Hex.java.
+ *
+ * @param ch A character to convert to an integer digit
+ * @param idx The index of the character in the source
+ * @return An integer
+ * @throws TableException Thrown if ch is an illegal hex character
+ */
+ private static int toDigit(final char ch, final int idx) throws
TableException {
+ final int digit = Character.digit(ch, 16);
+ if (digit == -1) {
+ throw new TableException(
+ "Illegal hexadecimal character: [" + ch + "] at index: ["
+ idx + "]");
+ }
+ return digit;
+ }
+
//
--------------------------------------------------------------------------------------------
// Java String Repetition
//
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/ArrayToStringCastRule.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/ArrayToStringCastRule.java
index 7df8b76..33d7209 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/ArrayToStringCastRule.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/ArrayToStringCastRule.java
@@ -148,7 +148,7 @@ class ArrayToStringCastRule extends
AbstractNullAwareCodeGeneratorCastRule<Array
// Break if the target length is
already exceeded
loopBodyWriter.ifStmt(
stringExceedsLength(builderTerm, length),
- thenBodyWriter ->
thenBodyWriter.stmt("break"));
+
CastRuleUtils.CodeWriter::breakStmt);
}
loopBodyWriter
// Write the comma
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/BinaryToStringCastRule.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/BinaryToStringCastRule.java
index 126e3c0..566584c 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/BinaryToStringCastRule.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/BinaryToStringCastRule.java
@@ -20,17 +20,21 @@ package org.apache.flink.table.planner.functions.casting;
import org.apache.flink.table.types.logical.LogicalType;
import org.apache.flink.table.types.logical.LogicalTypeFamily;
+import org.apache.flink.table.types.logical.utils.LogicalTypeChecks;
+import org.apache.flink.table.utils.EncodingUtils;
import java.nio.charset.StandardCharsets;
+import static org.apache.flink.table.planner.codegen.CodeGenUtils.newName;
+import static
org.apache.flink.table.planner.codegen.calls.BuiltInMethods.BINARY_STRING_DATA_FROM_STRING;
import static
org.apache.flink.table.planner.functions.casting.CastRuleUtils.accessStaticField;
import static
org.apache.flink.table.planner.functions.casting.CastRuleUtils.constructorCall;
-import static org.apache.flink.table.types.logical.VarCharType.STRING_TYPE;
+import static
org.apache.flink.table.planner.functions.casting.CastRuleUtils.staticCall;
/**
* {@link LogicalTypeFamily#BINARY_STRING} to {@link
LogicalTypeFamily#CHARACTER_STRING} cast rule.
*/
-class BinaryToStringCastRule extends AbstractCharacterFamilyTargetRule<byte[]>
{
+class BinaryToStringCastRule extends
AbstractNullAwareCodeGeneratorCastRule<byte[], String> {
static final BinaryToStringCastRule INSTANCE = new
BinaryToStringCastRule();
@@ -38,17 +42,78 @@ class BinaryToStringCastRule extends
AbstractCharacterFamilyTargetRule<byte[]> {
super(
CastRulePredicate.builder()
.input(LogicalTypeFamily.BINARY_STRING)
- .target(STRING_TYPE)
+ .target(LogicalTypeFamily.CHARACTER_STRING)
.build());
}
+ /* Example generated code
+
+ isNull$0 = _myInputIsNull;
+ if (!isNull$0) {
+ java.lang.String hexString$0;
+ hexString$0 = org.apache.flink.table.utils.EncodingUtils.hex(_myInput);
+ java.lang.String resultString$152;
+ resultString$152 = hexString$0.toString();
+ if (hexString$0.length() > 3) {
+ resultString$152 = hexString$0.substring(0,
java.lang.Math.min(hexString$0.length(), 3));
+ } else {
+ if (resultString$1.length() < 12) {
+ int padLength$3;
+ padLength$3 = 12 - resultString$152.length();
+ java.lang.StringBuilder sbPadding$4;
+ sbPadding$4 = new java.lang.StringBuilder();
+ for (int i$5 = 0; i$5 < padLength$3; i$5++) {
+ sbPadding$4.append(" ");
+ }
+ resultString$152 = resultString$152 + sbPadding$4.toString();
+ }
+ }
+ result$1 =
org.apache.flink.table.data.binary.BinaryStringData.fromString(resultString$152);
+ isNull$0 = result$1 == null;
+ } else {
+ result$1 =
org.apache.flink.table.data.binary.BinaryStringData.EMPTY_UTF8;
+ }
+
+ */
+
@Override
- public String generateStringExpression(
+ protected String generateCodeBlockInternal(
CodeGeneratorCastRule.Context context,
String inputTerm,
+ String returnVariable,
LogicalType inputLogicalType,
LogicalType targetLogicalType) {
- return constructorCall(
- String.class, inputTerm,
accessStaticField(StandardCharsets.class, "UTF_8"));
+ final String resultStringTerm = newName("resultString");
+ CastRuleUtils.CodeWriter writer = new CastRuleUtils.CodeWriter();
+ if (context.legacyBehaviour()) {
+ writer.declStmt(String.class, resultStringTerm)
+ .assignStmt(
+ resultStringTerm,
+ constructorCall(
+ String.class,
+ inputTerm,
+ accessStaticField(StandardCharsets.class,
"UTF_8")));
+ } else {
+ final int length = LogicalTypeChecks.getLength(targetLogicalType);
+
+ final String hexStringTerm = newName("hexString");
+ writer.declStmt(String.class, hexStringTerm)
+ .assignStmt(hexStringTerm, staticCall(EncodingUtils.class,
"hex", inputTerm));
+ writer =
+ CharVarCharTrimPadCastRule.padAndTrimStringIfNeeded(
+ writer,
+ targetLogicalType,
+ context.legacyBehaviour(),
+ length,
+ resultStringTerm,
+ hexStringTerm);
+ }
+ return writer
+ // Assign the result value
+ .assignStmt(
+ returnVariable,
+ CastRuleUtils.staticCall(
+ BINARY_STRING_DATA_FROM_STRING(),
resultStringTerm))
+ .toString();
}
}
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/CastRuleUtils.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/CastRuleUtils.java
index 8bbba2b..df2e08e 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/CastRuleUtils.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/CastRuleUtils.java
@@ -78,8 +78,12 @@ final class CastRuleUtils {
return className(clazz) + "." + fieldName;
}
- static String arrayLength(String instanceTerm) {
- return instanceTerm + ".length";
+ static String arrayLength(String arrayTerm) {
+ return arrayTerm + ".length";
+ }
+
+ static String arrayElement(String arrayTerm, String indexTerm) {
+ return arrayTerm + "[" + indexTerm + "]";
}
static String ternaryOperator(String condition, String ifTrue, String
ifFalse) {
@@ -233,6 +237,11 @@ final class CastRuleUtils {
return this;
}
+ public CodeWriter breakStmt() {
+ builder.append("break;\n");
+ return this;
+ }
+
public CodeWriter ifStmt(String condition, Consumer<CodeWriter>
bodyWriterConsumer) {
final CodeWriter innerWriter = new CodeWriter();
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/MapAndMultisetToStringCastRule.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/MapAndMultisetToStringCastRule.java
index 23c015d..24ba056 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/MapAndMultisetToStringCastRule.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/MapAndMultisetToStringCastRule.java
@@ -219,7 +219,7 @@ class MapAndMultisetToStringCastRule
// exceeded
.ifStmt(
stringExceedsLength(builderTerm, length),
- thenBodyWriter ->
thenBodyWriter.stmt("break"));
+
CastRuleUtils.CodeWriter::breakStmt);
}
loopBodyWriter
// Write the comma
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/StringToBinaryCastRule.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/StringToBinaryCastRule.java
index a0c7bb0..ad95a41 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/StringToBinaryCastRule.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/StringToBinaryCastRule.java
@@ -22,13 +22,14 @@ import org.apache.flink.table.data.StringData;
import org.apache.flink.table.types.logical.LogicalType;
import org.apache.flink.table.types.logical.LogicalTypeFamily;
import org.apache.flink.table.types.logical.utils.LogicalTypeChecks;
+import org.apache.flink.table.utils.EncodingUtils;
import static org.apache.flink.table.codesplit.CodeSplitUtil.newName;
import static
org.apache.flink.table.planner.functions.casting.BinaryToBinaryCastRule.couldPad;
-import static
org.apache.flink.table.planner.functions.casting.BinaryToBinaryCastRule.couldTrim;
import static
org.apache.flink.table.planner.functions.casting.BinaryToBinaryCastRule.trimOrPadByteArray;
import static
org.apache.flink.table.planner.functions.casting.CastRuleUtils.arrayLength;
import static
org.apache.flink.table.planner.functions.casting.CastRuleUtils.methodCall;
+import static
org.apache.flink.table.planner.functions.casting.CastRuleUtils.staticCall;
/**
* {@link LogicalTypeFamily#CHARACTER_STRING} to {@link
LogicalTypeFamily#BINARY_STRING} cast rule.
@@ -45,7 +46,7 @@ class StringToBinaryCastRule extends
AbstractNullAwareCodeGeneratorCastRule<Stri
.build());
}
- /* Example generated code for BINARY(2):
+ /* Example generated code for VARBINARY(2):
// legacy behavior
isNull$0 = _myInputIsNull;
@@ -59,9 +60,12 @@ class StringToBinaryCastRule extends
AbstractNullAwareCodeGeneratorCastRule<Stri
// new behavior
isNull$0 = _myInputIsNull;
if (!isNull$0) {
- byte[] byteArrayTerm$0 = _myInput.toBytes();
+ java.lang.String hexStringTerm$10 = _myInput.toString();
+ byte[] byteArrayTerm$0 =
org.apache.flink.table.utils.EncodingUtils.decodeHex(hexStringTerm$10);
if (byteArrayTerm$0.length <= 2) {
- result$1 = byteArrayTerm$0;
+ // If could pad
+ result$1 = java.util.Arrays.copyOf(byteArrayTerm$0, 2);
+ // result$1 = byteArrayTerm$0 // If could not pad
} else {
result$1 = java.util.Arrays.copyOf(byteArrayTerm$0, 2);
}
@@ -79,18 +83,21 @@ class StringToBinaryCastRule extends
AbstractNullAwareCodeGeneratorCastRule<Stri
String returnVariable,
LogicalType inputLogicalType,
LogicalType targetLogicalType) {
- final int targetLength =
LogicalTypeChecks.getLength(targetLogicalType);
-
- final String byteArrayTerm = newName("byteArrayTerm");
-
- if (context.legacyBehaviour()
- || !(couldTrim(targetLength) || couldPad(targetLogicalType,
targetLength))) {
+ if (context.legacyBehaviour()) {
return new CastRuleUtils.CodeWriter()
.assignStmt(returnVariable, methodCall(inputTerm,
"toBytes"))
.toString();
} else {
+ final int targetLength =
LogicalTypeChecks.getLength(targetLogicalType);
+ final String byteArrayTerm = newName("byteArrayTerm");
+ final String hexStringTerm = newName("hexStringTerm");
+
return new CastRuleUtils.CodeWriter()
- .declStmt(byte[].class, byteArrayTerm,
methodCall(inputTerm, "toBytes"))
+ .declStmt(String.class, hexStringTerm,
methodCall(inputTerm, "toString"))
+ .declStmt(
+ byte[].class,
+ byteArrayTerm,
+ staticCall(EncodingUtils.class, "decodeHex",
hexStringTerm))
.ifStmt(
arrayLength(byteArrayTerm) + " <= " + targetLength,
thenWriter -> {
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java
index 42c042d..a0449a8 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java
@@ -157,11 +157,11 @@ public class CastFunctionITCase extends
BuiltInFunctionTestBase {
.fromCase(STRING(), "Apache Flink", "Apache Flink")
.fromCase(STRING(), null, null)
.fromCase(BOOLEAN(), true, "TRUE")
- .fromCase(BINARY(2), DEFAULT_BINARY, "\u0000\u0001")
- .fromCase(BINARY(3), DEFAULT_BINARY,
"\u0000\u0001\u0000")
- .fromCase(VARBINARY(3), DEFAULT_VARBINARY,
"\u0000\u0001\u0002")
- .fromCase(VARBINARY(5), DEFAULT_VARBINARY,
"\u0000\u0001\u0002")
- .fromCase(BYTES(), DEFAULT_BYTES,
"\u0000\u0001\u0002\u0003\u0004")
+ .fromCase(BINARY(2), DEFAULT_BINARY, "0001")
+ .fromCase(BINARY(3), DEFAULT_BINARY, "000100")
+ .fromCase(VARBINARY(3), DEFAULT_VARBINARY, "000102")
+ .fromCase(VARBINARY(5), DEFAULT_VARBINARY, "000102")
+ .fromCase(BYTES(), DEFAULT_BYTES, "0001020304")
.fromCase(DECIMAL(4, 3), 9.87, "9.870")
.fromCase(DECIMAL(10, 5), 1, "1.00000")
.fromCase(
@@ -298,11 +298,11 @@ public class CastFunctionITCase extends
BuiltInFunctionTestBase {
.build(),
CastTestSpecBuilder.testCastTo(BINARY(2))
.fromCase(BINARY(5), null, null)
- .fromCase(CHAR(3), "foo", new byte[] {102, 111})
- .fromCase(VARCHAR(5), "Flink", new byte[] {70, 108})
- .fromCase(STRING(), "Apache", new byte[] {65, 112})
- .fromCase(VARCHAR(5), "f", new byte[] {102, 0})
- .fromCase(STRING(), "f", new byte[] {102, 0})
+ .fromCase(CHAR(4), "666F", new byte[] {102, 111})
+ .fromCase(VARCHAR(8), "666f", new byte[] {102, 111})
+ .fromCase(STRING(), "AAbbcCdD", new byte[] {-86, -69})
+ .fromCase(VARCHAR(4), "FC", new byte[] {-4, 0})
+ .fromCase(STRING(), "df", new byte[] {-33, 0})
// Not supported - no fix
.fail(BOOLEAN(), true)
//
@@ -336,9 +336,9 @@ public class CastFunctionITCase extends
BuiltInFunctionTestBase {
.build(),
CastTestSpecBuilder.testCastTo(VARBINARY(4))
.fromCase(VARBINARY(5), null, null)
- .fromCase(CHAR(3), "foo", new byte[] {102, 111, 111})
- .fromCase(VARCHAR(5), "Flink", new byte[] {70, 108,
105, 110})
- .fromCase(STRING(), "Apache", new byte[] {65, 112, 97,
99})
+ .fromCase(CHAR(4), "666F", new byte[] {102, 111})
+ .fromCase(VARCHAR(8), "666f", new byte[] {102, 111})
+ .fromCase(STRING(), "AAbbCcDdEe", new byte[] {-86,
-69, -52, -35})
// Not supported - no fix
.fail(BOOLEAN(), true)
//
@@ -370,9 +370,9 @@ public class CastFunctionITCase extends
BuiltInFunctionTestBase {
.build(),
CastTestSpecBuilder.testCastTo(BYTES())
.fromCase(BYTES(), null, null)
- .fromCase(CHAR(3), "foo", new byte[] {102, 111, 111})
- .fromCase(VARCHAR(5), "Flink", new byte[] {70, 108,
105, 110, 107})
- .fromCase(STRING(), "Apache", new byte[] {65, 112, 97,
99, 104, 101})
+ .fromCase(CHAR(4), "666f", new byte[] {102, 111})
+ .fromCase(VARCHAR(8), "666F", new byte[] {102, 111})
+ .fromCase(STRING(), "aaBBCcDdEe", new byte[] {-86,
-69, -52, -35, -18})
// Not supported - no fix
.fail(BOOLEAN(), true)
//
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionMiscITCase.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionMiscITCase.java
index 3ca2021..54050ad 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionMiscITCase.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionMiscITCase.java
@@ -198,14 +198,14 @@ public class CastFunctionMiscITCase extends
BuiltInFunctionTestBase {
.onFieldsWithData("foo")
.testSqlResult(
"CAST(CAST(x'68656C6C6F20636F6465' AS
BINARY(10)) AS VARCHAR)",
- "hello code",
+ "68656c6c6f20636f6465",
STRING().notNull()),
TestSpec.forFunction(
BuiltInFunctionDefinitions.CAST, "test the
x'....' binary syntax")
.onFieldsWithData("foo")
.testSqlResult(
"CAST(CAST(x'68656C6C6F2063617374' AS
BINARY(10)) AS VARCHAR)",
- "hello cast",
+ "68656c6c6f2063617374",
STRING().notNull()));
}
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/casting/CastRulesTest.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/casting/CastRulesTest.java
index 48d2b75..e82436a 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/casting/CastRulesTest.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/casting/CastRulesTest.java
@@ -490,19 +490,23 @@ class CastRulesTest {
.fromCase(BOOLEAN(), false, fromString("FALSE"))
.fromCaseLegacy(BOOLEAN(), true, fromString("true"))
.fromCaseLegacy(BOOLEAN(), false, fromString("false"))
- .fromCase(BINARY(2), new byte[] {0, 1},
fromString("\u0000\u0001"))
- .fromCase(
+ .fromCase(BINARY(2), new byte[] {0, 1},
fromString("0001"))
+ .fromCaseLegacy(BINARY(2), new byte[] {0, 1},
fromString("\u0000\u0001"))
+ .fromCase(VARBINARY(3), new byte[] {0, 1, 2},
fromString("000102"))
+ .fromCaseLegacy(
VARBINARY(3),
new byte[] {0, 1, 2},
fromString("\u0000\u0001\u0002"))
- .fromCase(
- VARBINARY(5),
- new byte[] {0, 1, 2},
- fromString("\u0000\u0001\u0002"))
+ .fromCase(VARBINARY(5), new byte[] {0, -1, -2},
fromString("00fffe"))
+ .fromCaseLegacy(VARBINARY(5), new byte[] {102, 111,
111}, fromString("foo"))
.fromCase(
BYTES(),
- new byte[] {0, 1, 2, 3, 4},
- fromString("\u0000\u0001\u0002\u0003\u0004"))
+ new byte[] {-123, 43, -4, 125, 5},
+ fromString("852bfc7d05"))
+ .fromCaseLegacy(
+ BYTES(), new byte[] {70, 108, 105, 110, 107},
fromString("Flink"))
+ .fromCase(BOOLEAN(), true,
StringData.fromString("TRUE"))
+ .fromCase(BOOLEAN(), false,
StringData.fromString("FALSE"))
.fromCase(
DECIMAL(4, 3),
fromBigDecimal(new BigDecimal("9.87"), 4, 3),
@@ -660,28 +664,12 @@ class CastRulesTest {
.fromCaseLegacy(BOOLEAN(), true, fromString("true"))
.fromCase(BOOLEAN(), false, fromString("FALSE "))
.fromCaseLegacy(BOOLEAN(), false, fromString("false"))
- .fromCase(
- BINARY(3),
- new byte[] {0, 1, 2},
- fromString("\u0000\u0001\u0002 "))
- .fromCaseLegacy(
- BINARY(3), new byte[] {0, 1, 2},
fromString("\u0000\u0001\u0002"))
- .fromCase(
- VARBINARY(4),
- new byte[] {0, 1, 2, 3},
- fromString("\u0000\u0001\u0002\u0003 "))
- .fromCaseLegacy(
- VARBINARY(4),
- new byte[] {0, 1, 2, 3},
- fromString("\u0000\u0001\u0002\u0003"))
- .fromCase(
- BYTES(),
- new byte[] {0, 1, 2, 3, 4},
- fromString("\u0000\u0001\u0002\u0003\u0004 "))
- .fromCaseLegacy(
- BYTES(),
- new byte[] {0, 1, 2, 3, 4},
- fromString("\u0000\u0001\u0002\u0003\u0004"))
+ .fromCase(BINARY(1), new byte[] {-12}, fromString("f4
"))
+ .fromCaseLegacy(BINARY(1), new byte[] {102},
fromString("f"))
+ .fromCase(VARBINARY(1), new byte[] {23},
fromString("17 "))
+ .fromCaseLegacy(VARBINARY(1), new byte[] {33},
fromString("\u0021"))
+ .fromCase(BYTES(), new byte[] {32}, fromString("20
"))
+ .fromCaseLegacy(BYTES(), new byte[] {32}, fromString("
"))
.fromCase(TINYINT(), (byte) -125, fromString("-125 "))
.fromCaseLegacy(TINYINT(), (byte) -125,
fromString("-125"))
.fromCase(SMALLINT(), (short) 32767, fromString("32767
"))
@@ -700,6 +688,22 @@ class CastRulesTest {
.fromCaseLegacy(INTERVAL(MONTH()), 5,
fromString("+0-05")),
CastTestSpecBuilder.testCastTo(CHAR(12))
.fromCase(
+ BINARY(4),
+ new byte[] {-12, 32, 46, -72},
+ fromString("f4202eb8 "))
+ .fromCaseLegacy(
+ BINARY(4),
+ new byte[] {1, 11, 111, 2},
+ fromString("\u0001\u000B\u006F\u0002"))
+ .fromCase(VARBINARY(4), new byte[] {1, 11, 22},
fromString("010b16 "))
+ .fromCaseLegacy(
+ VARBINARY(4),
+ new byte[] {1, 11, 22},
+ fromString("\u0001\u000B\u0016"))
+ .fromCase(BYTES(), new byte[] {1, 11},
fromString("010b "))
+ .fromCaseLegacy(
+ BYTES(), new byte[] {1, 11, 111},
fromString("\u0001\u000B\u006F"))
+ .fromCase(
ARRAY(INT()),
new GenericArrayData(new int[] {-1, 2, 3}),
fromString("[-1, 2, 3] "))
@@ -785,26 +789,17 @@ class CastRulesTest {
.fromCaseLegacy(BOOLEAN(), true, fromString("true"))
.fromCase(BOOLEAN(), false, fromString("FAL"))
.fromCaseLegacy(BOOLEAN(), false, fromString("false"))
- .fromCase(
- BINARY(5),
- new byte[] {0, 1, 2, 3, 4},
- fromString("\u0000\u0001\u0002"))
+ .fromCase(BINARY(5), new byte[] {0, 1, 2, 3, 4},
fromString("000"))
.fromCaseLegacy(
BINARY(5),
new byte[] {0, 1, 2, 3, 4},
fromString("\u0000\u0001\u0002\u0003\u0004"))
- .fromCase(
- VARBINARY(5),
- new byte[] {0, 1, 2, 3, 4},
- fromString("\u0000\u0001\u0002"))
+ .fromCase(VARBINARY(5), new byte[] {0, 1, 2, 3, 4},
fromString("000"))
.fromCaseLegacy(
VARBINARY(5),
new byte[] {0, 1, 2, 3, 4},
fromString("\u0000\u0001\u0002\u0003\u0004"))
- .fromCase(
- BYTES(),
- new byte[] {0, 1, 2, 3, 4},
- fromString("\u0000\u0001\u0002"))
+ .fromCase(BYTES(), new byte[] {0, 1, 2, 3, 4},
fromString("000"))
.fromCaseLegacy(
BYTES(),
new byte[] {0, 1, 2, 3, 4},
@@ -958,24 +953,30 @@ class CastRulesTest {
.fromCase(DOUBLE(), 0.0d, false)
.fromCase(DOUBLE(), -0.12345678d, true),
CastTestSpecBuilder.testCastTo(BINARY(4))
- .fromCase(CHAR(3), fromString("foo"), new byte[] {102,
111, 111, 0})
+ .fromCase(CHAR(4), fromString("66"), new byte[] {102,
0, 0, 0})
.fromCaseLegacy(CHAR(3), fromString("foo"), new byte[]
{102, 111, 111})
- .fromCase(CHAR(1), fromString("f"), new byte[] {102,
0, 0, 0})
+ .fromCase(CHAR(10), fromString("66A2"), new byte[]
{102, -94, 0, 0})
.fromCaseLegacy(CHAR(1), fromString("f"), new byte[]
{102})
- .fromCase(CHAR(3), fromString("f"), new byte[] {102,
0, 0, 0})
+ .fromCase(CHAR(16), fromString("12f4aBc7"), new byte[]
{18, -12, -85, -57})
.fromCaseLegacy(CHAR(3), fromString("f"), new byte[]
{102})
- .fromCase(VARCHAR(5), fromString("Flink"), new byte[]
{70, 108, 105, 110})
+ .fromCase(VARCHAR(8), fromString("bACd"), new byte[]
{-70, -51, 0, 0})
.fromCaseLegacy(
VARCHAR(5),
fromString("Flink"),
new byte[] {70, 108, 105, 110, 107})
- .fromCase(STRING(), fromString("Apache"), new byte[]
{65, 112, 97, 99})
+ .fromCase(
+ STRING(),
+ fromString("12f4ABc71232"),
+ new byte[] {18, -12, -85, -57})
.fromCaseLegacy(
STRING(),
fromString("Apache"),
new byte[] {65, 112, 97, 99, 104, 101})
- .fromCase(STRING(), fromString("bar"), new byte[] {98,
97, 114, 0})
+ .fromCase(STRING(), fromString("12F4ab"), new byte[]
{18, -12, -85, 0})
.fromCaseLegacy(STRING(), fromString("bar"), new
byte[] {98, 97, 114})
+ .fail(STRING(), fromString("123"),
TableException.class)
+ .fail(STRING(), fromString("12P9"),
TableException.class)
+ .fail(STRING(), fromString("12 A9"),
TableException.class)
.fromCase(BINARY(2), new byte[] {1, 2}, new byte[] {1,
2, 0, 0})
.fromCaseLegacy(BINARY(2), new byte[] {1, 2}, new
byte[] {1, 2})
.fromCase(VARBINARY(3), new byte[] {1, 2, 3}, new
byte[] {1, 2, 3, 0})
@@ -983,17 +984,24 @@ class CastRulesTest {
.fromCase(BYTES(), new byte[] {1, 2, 3}, new byte[]
{1, 2, 3, 0})
.fromCaseLegacy(BYTES(), new byte[] {1, 2, 3}, new
byte[] {1, 2, 3}),
CastTestSpecBuilder.testCastTo(VARBINARY(4))
- .fromCase(CHAR(3), fromString("foo"), new byte[] {102,
111, 111})
+ .fromCase(CHAR(4), fromString("c9"), new byte[] {-55})
+ .fromCaseLegacy(CHAR(3), fromString("foo"), new byte[]
{102, 111, 111})
+ .fromCase(VARCHAR(8), fromString("7de2"), new byte[]
{125, -30})
.fromCaseLegacy(
VARCHAR(5),
fromString("Flink"),
new byte[] {70, 108, 105, 110, 107})
- .fromCase(VARCHAR(5), fromString("Flink"), new byte[]
{70, 108, 105, 110})
- .fromCase(STRING(), fromString("Apache"), new byte[]
{65, 112, 97, 99})
+ .fromCase(
+ STRING(),
+ fromString("12F4abC71232"),
+ new byte[] {18, -12, -85, -57})
.fromCaseLegacy(
STRING(),
fromString("Apache"),
new byte[] {65, 112, 97, 99, 104, 101})
+ .fail(STRING(), fromString("123"),
TableException.class)
+ .fail(STRING(), fromString("12P9"),
TableException.class)
+ .fail(STRING(), fromString("12 A9"),
TableException.class)
// We assume that the input length is respected,
therefore, no trimming is
// applied
.fromCase(BINARY(2), new byte[] {1, 2, 3, 4, 5}, new
byte[] {1, 2, 3, 4, 5})
@@ -1008,15 +1016,24 @@ class CastRulesTest {
new byte[] {1, 2, 3, 4, 5},
new byte[] {1, 2, 3, 4, 5}),
CastTestSpecBuilder.testCastTo(BYTES())
- .fromCase(CHAR(3), fromString("foo"), new byte[] {102,
111, 111})
- .fromCase(
+ .fromCase(CHAR(4), fromString("9C"), new byte[] {-100})
+ .fromCaseLegacy(CHAR(3), fromString("foo"), new byte[]
{102, 111, 111})
+ .fromCase(VARCHAR(8), fromString("3ee3"), new byte[]
{62, -29})
+ .fromCaseLegacy(
VARCHAR(5),
fromString("Flink"),
new byte[] {70, 108, 105, 110, 107})
.fromCase(
STRING(),
+ fromString("AAbbCcDdff"),
+ new byte[] {-86, -69, -52, -35, -1})
+ .fromCaseLegacy(
+ STRING(),
fromString("Apache"),
- new byte[] {65, 112, 97, 99, 104, 101}),
+ new byte[] {65, 112, 97, 99, 104, 101})
+ .fail(STRING(), fromString("123"),
TableException.class)
+ .fail(STRING(), fromString("12P9"),
TableException.class)
+ .fail(STRING(), fromString("12 A9"),
TableException.class),
CastTestSpecBuilder.testCastTo(DECIMAL(5, 3))
.fail(CHAR(3), fromString("foo"), TableException.class)
.fail(VARCHAR(5), fromString("Flink"),
TableException.class)
diff --git
a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala
b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala
index 10f57d3..e09918b 100644
---
a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala
+++
b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala
@@ -3862,18 +3862,7 @@ class ScalarFunctionsTest extends ScalarTypesTestBase {
"SHA2('test', f44)",
expectedSha256)
- // bytes test
- testSqlApi("MD5(cast('test' as varbinary))", expectedMd5)
- testSqlApi("SHA1(cast('test' as varbinary))", expectedSha1)
- testSqlApi("SHA224(cast('test' as varbinary))", expectedSha224)
- testSqlApi("SHA2(cast('test' as varbinary), 224)", expectedSha224)
- testSqlApi("SHA256(cast('test' as varbinary))", expectedSha256)
- testSqlApi("SHA2(cast('test' as varbinary), 256)", expectedSha256)
- testSqlApi("SHA384(cast('test' as varbinary))", expectedSha384)
- testSqlApi("SHA2(cast('test' as varbinary), 384)", expectedSha384)
- testSqlApi("SHA512(cast('test' as varbinary))", expectedSha512)
- testSqlApi("SHA2(cast('test' as varbinary), 512)", expectedSha512)
-
+ // null test
testSqlApi("MD5(cast(null as varbinary))", "NULL")
testSqlApi("SHA1(cast(null as varbinary))", "NULL")
testSqlApi("SHA224(cast(null as varbinary))", "NULL")
@@ -4157,7 +4146,7 @@ class ScalarFunctionsTest extends ScalarTypesTestBase {
// the answer BINARY will cast to STRING in ExpressionTestBase.scala
testSqlApi(
"IF(f7 < 5, f53, f54)",
- "hello world") // hello world
+ "68656c6c6f20776f726c64") // hello world
// test DATE, DATE
testSqlApi(
@@ -4363,6 +4352,8 @@ class ScalarFunctionsTest extends ScalarTypesTestBase {
"2021-04-06 11:05:30")
testSqlApi(s"IFNULL(CAST(INTERVAL '2' YEAR AS VARCHAR(20)), $str2)",
"+2-00")
testSqlApi(s"IFNULL(CAST(INTERVAL '2' DAY AS VARCHAR(20)), $str2)", "+2
00:00:00.000")
- testSqlApi(s"IFNULL(CAST(f53 AS VARCHAR(100)), $str2)", "hello world")
+ testSqlApi(
+ s"IFNULL(CAST(f53 AS VARCHAR(100)), $str2)",
+ "68656c6c6f20776f726c64")
}
}
