rdblue commented on code in PR #7886:
URL: https://github.com/apache/iceberg/pull/7886#discussion_r1264205722
##########
spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestSparkV2Filters.java:
##########
@@ -373,4 +386,371 @@ public void testNotIn() {
Expressions.and(Expressions.notNull("col"), Expressions.notIn("col",
1, 2));
Assert.assertEquals("Expressions should match", expected.toString(),
actual.toString());
}
+
+ @Test
+ public void testDateToYears() {
+ ScalarFunction<Integer> dateToYearsFunc = new
YearsFunction.DateToYearsFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ dateToYearsFunc.name(),
+ dateToYearsFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(udf, Expressions.year("col1"), dateToYears("2023-06-25"),
DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTsToYears() {
+ ScalarFunction<Integer> tsToYearsFunc = new
YearsFunction.TimestampToYearsFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ tsToYearsFunc.name(),
+ tsToYearsFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(
+ udf,
+ Expressions.year("col1"),
+ timestampToYears("2023-12-03T10:15:30+01:00"),
+ DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTsNtzToYears() {
+ ScalarFunction<Integer> tsNtzToYearsFunc = new
YearsFunction.TimestampNtzToYearsFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ tsNtzToYearsFunc.name(),
+ tsNtzToYearsFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(
+ udf,
+ Expressions.year("col1"),
+ timestampNtzToYears("2023-06-25T13:15:30"),
+ DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testDateToMonths() {
+ ScalarFunction<Integer> dateToMonthsFunc = new
MonthsFunction.DateToMonthsFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ dateToMonthsFunc.name(),
+ dateToMonthsFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(udf, Expressions.month("col1"), dateToMonths("2023-06-25"),
DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTsToMonths() {
+ ScalarFunction<Integer> tsToMonthsFunc = new
MonthsFunction.TimestampToMonthsFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ tsToMonthsFunc.name(),
+ tsToMonthsFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(
+ udf,
+ Expressions.month("col1"),
+ timestampToMonths("2023-12-03T10:15:30+01:00"),
+ DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTsNtzToMonths() {
+ ScalarFunction<Integer> tsNtzToMonthsFunc = new
MonthsFunction.TimestampNtzToMonthsFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ tsNtzToMonthsFunc.name(),
+ tsNtzToMonthsFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(
+ udf,
+ Expressions.month("col1"),
+ timestampNtzToMonths("2023-12-03T10:15:30"),
+ DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testDateToDays() {
+ ScalarFunction<Integer> dateToDayFunc = new
DaysFunction.DateToDaysFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ dateToDayFunc.name(),
+ dateToDayFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(udf, Expressions.day("col1"), dateToDays("2023-06-25"),
DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTsToDays() {
+ ScalarFunction<Integer> tsToDaysFunc = new
DaysFunction.TimestampToDaysFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ tsToDaysFunc.name(),
+ tsToDaysFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(
+ udf,
+ Expressions.day("col1"),
+ timestampToDays("2023-12-03T10:15:30+01:00"),
+ DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTsNtzToDays() {
+ ScalarFunction<Integer> tsNtzToDaysFunc = new
DaysFunction.TimestampNtzToDaysFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ tsNtzToDaysFunc.name(),
+ tsNtzToDaysFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(
+ udf,
+ Expressions.day("col1"),
+ timestampNtzToDays("2023-12-03T10:15:30"),
+ DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTsToHours() {
+ ScalarFunction<Integer> tsToHourFunc = new
HoursFunction.TimestampToHoursFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ tsToHourFunc.name(),
+ tsToHourFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(
+ udf,
+ Expressions.hour("col1"),
+ timestampToHours("2023-12-03T10:15:30+01:00"),
+ DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTsNtzToHours() {
+ ScalarFunction<Integer> tsNtzToHourFunc = new
HoursFunction.TimestampNtzToHoursFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ tsNtzToHourFunc.name(),
+ tsNtzToHourFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(
+ udf,
+ Expressions.hour("col1"),
+ timestampNtzToHours("2023-12-03T10:15:30"),
+ DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testBucket() {
+ ScalarFunction<Integer> bucketInt = new
BucketFunction.BucketInt(DataTypes.IntegerType);
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ bucketInt.name(),
+ bucketInt.canonicalName(),
+ expressions(
+ LiteralValue.apply(4, DataTypes.IntegerType),
FieldReference.apply("col1")));
+ testUDF(udf, Expressions.bucket("col1", 4), 2, DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTruncate() {
+ ScalarFunction<UTF8String> truncate = new
TruncateFunction.TruncateString();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ truncate.name(),
+ truncate.canonicalName(),
+ expressions(
+ LiteralValue.apply(6, DataTypes.IntegerType),
FieldReference.apply("col1")));
+ testUDF(udf, Expressions.truncate("col1", 6), "prefix",
DataTypes.StringType);
+ }
+
+ @Test
+ public void testUnsupportedUDFConvert() {
+ ScalarFunction<UTF8String> icebergVersionFunc =
+ (ScalarFunction<UTF8String>) new IcebergVersionFunction().bind(new
StructType());
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ icebergVersionFunc.name(),
+ icebergVersionFunc.canonicalName(),
+ new org.apache.spark.sql.connector.expressions.Expression[] {});
+ LiteralValue literalValue = new LiteralValue("1.3.0",
DataTypes.StringType);
+ Predicate predicate = new Predicate("=", expressions(udf, literalValue));
+
+ Expression icebergExpr = SparkV2Filters.convert(predicate);
+ Assertions.assertThat(icebergExpr).isNull();
+ }
+
+ private <T> void testUDF(
+ org.apache.spark.sql.connector.expressions.Expression expression,
+ UnboundTerm<T> item,
+ T value,
+ DataType dataType) {
+ org.apache.spark.sql.connector.expressions.Expression[] attrOnly =
expressions(expression);
+
+ LiteralValue literalValue = new LiteralValue(value, dataType);
+ org.apache.spark.sql.connector.expressions.Expression[] attrAndValue =
+ expressions(expression, literalValue);
+ org.apache.spark.sql.connector.expressions.Expression[] valueAndAttr =
+ expressions(literalValue, expression);
+
+ Predicate isNull = new Predicate("IS_NULL", attrOnly);
+ Expression expectedIsNull = Expressions.isNull(item);
+ Expression actualIsNull = SparkV2Filters.convert(isNull);
+
Assertions.assertThat(actualIsNull.toString()).isEqualTo(expectedIsNull.toString());
+
+ Predicate isNotNull = new Predicate("IS_NOT_NULL", attrOnly);
+ Expression expectedIsNotNull = Expressions.notNull(item);
+ Expression actualIsNotNull = SparkV2Filters.convert(isNotNull);
+
Assertions.assertThat(actualIsNotNull.toString()).isEqualTo(expectedIsNotNull.toString());
+
+ Predicate lt1 = new Predicate("<", attrAndValue);
+ Expression expectedLt1 = Expressions.lessThan(item, value);
+ Expression actualLt1 = SparkV2Filters.convert(lt1);
+
Assertions.assertThat(actualLt1.toString()).isEqualTo(expectedLt1.toString());
+
+ Predicate lt2 = new Predicate("<", valueAndAttr);
+ Expression expectedLt2 = Expressions.greaterThan(item, value);
+ Expression actualLt2 = SparkV2Filters.convert(lt2);
+
Assertions.assertThat(actualLt2.toString()).isEqualTo(expectedLt2.toString());
+
+ Predicate ltEq1 = new Predicate("<=", attrAndValue);
+ Expression expectedLtEq1 = Expressions.lessThanOrEqual(item, value);
+ Expression actualLtEq1 = SparkV2Filters.convert(ltEq1);
+
Assertions.assertThat(actualLtEq1.toString()).isEqualTo(expectedLtEq1.toString());
+
+ Predicate ltEq2 = new Predicate("<=", valueAndAttr);
+ Expression expectedLtEq2 = Expressions.greaterThanOrEqual(item, value);
+ Expression actualLtEq2 = SparkV2Filters.convert(ltEq2);
+
Assertions.assertThat(actualLtEq2.toString()).isEqualTo(expectedLtEq2.toString());
+
+ Predicate gt1 = new Predicate(">", attrAndValue);
+ Expression expectedGt1 = Expressions.greaterThan(item, value);
+ Expression actualGt1 = SparkV2Filters.convert(gt1);
+
Assertions.assertThat(actualGt1.toString()).isEqualTo(expectedGt1.toString());
+
+ Predicate gt2 = new Predicate(">", valueAndAttr);
+ Expression expectedGt2 = Expressions.lessThan(item, value);
+ Expression actualGt2 = SparkV2Filters.convert(gt2);
+
Assertions.assertThat(actualGt2.toString()).isEqualTo(expectedGt2.toString());
+
+ Predicate gtEq1 = new Predicate(">=", attrAndValue);
+ Expression expectedGtEq1 = Expressions.greaterThanOrEqual(item, value);
+ Expression actualGtEq1 = SparkV2Filters.convert(gtEq1);
+
Assertions.assertThat(actualGtEq1.toString()).isEqualTo(expectedGtEq1.toString());
+
+ Predicate gtEq2 = new Predicate(">=", valueAndAttr);
+ Expression expectedGtEq2 = Expressions.lessThanOrEqual(item, value);
+ Expression actualGtEq2 = SparkV2Filters.convert(gtEq2);
+
Assertions.assertThat(actualGtEq2.toString()).isEqualTo(expectedGtEq2.toString());
+
+ Predicate eq1 = new Predicate("=", attrAndValue);
+ Expression expectedEq1 = Expressions.equal(item, value);
+ Expression actualEq1 = SparkV2Filters.convert(eq1);
+
Assertions.assertThat(actualEq1.toString()).isEqualTo(expectedEq1.toString());
+
+ Predicate eq2 = new Predicate("=", valueAndAttr);
+ Expression expectedEq2 = Expressions.equal(item, value);
+ Expression actualEq2 = SparkV2Filters.convert(eq2);
+
Assertions.assertThat(actualEq2.toString()).isEqualTo(expectedEq2.toString());
+
+ Predicate notEq1 = new Predicate("<>", attrAndValue);
+ Expression expectedNotEq1 = Expressions.notEqual(item, value);
+ Expression actualNotEq1 = SparkV2Filters.convert(notEq1);
+
Assertions.assertThat(actualNotEq1.toString()).isEqualTo(expectedNotEq1.toString());
+
+ Predicate notEq2 = new Predicate("<>", valueAndAttr);
+ Expression expectedNotEq2 = Expressions.notEqual(item, value);
+ Expression actualNotEq2 = SparkV2Filters.convert(notEq2);
+
Assertions.assertThat(actualNotEq2.toString()).isEqualTo(expectedNotEq2.toString());
+
+ Predicate eqNullSafe1 = new Predicate("<=>", attrAndValue);
+ Expression expectedEqNullSafe1 = Expressions.equal(item, value);
+ Expression actualEqNullSafe1 = SparkV2Filters.convert(eqNullSafe1);
+
Assertions.assertThat(actualEqNullSafe1.toString()).isEqualTo(expectedEqNullSafe1.toString());
+
+ Predicate eqNullSafe2 = new Predicate("<=>", valueAndAttr);
+ Expression expectedEqNullSafe2 = Expressions.equal(item, value);
+ Expression actualEqNullSafe2 = SparkV2Filters.convert(eqNullSafe2);
+
Assertions.assertThat(actualEqNullSafe2.toString()).isEqualTo(expectedEqNullSafe2.toString());
+
+ Predicate in = new Predicate("IN", attrAndValue);
+ Expression expectedIn = Expressions.in(item, value);
+ Expression actualIn = SparkV2Filters.convert(in);
+
Assertions.assertThat(actualIn.toString()).isEqualTo(expectedIn.toString());
+
+ Predicate and = new And(lt1, eq1);
+ Expression expectedAnd = Expressions.and(expectedLt1, expectedEq1);
+ Expression actualAnd = SparkV2Filters.convert(and);
+
Assertions.assertThat(actualAnd.toString()).isEqualTo(expectedAnd.toString());
+
+ org.apache.spark.sql.connector.expressions.Expression[] attrAndAttr =
+ expressions(expression, expression);
+ Predicate invalid = new Predicate("<", attrAndAttr);
+ Predicate andWithInvalidLeft = new And(invalid, eq1);
+ Expression convertedAnd = SparkV2Filters.convert(andWithInvalidLeft);
+ Assertions.assertThat(convertedAnd).isNull();
+
+ Predicate or = new Or(lt1, eq1);
+ Expression expectedOr = Expressions.or(expectedLt1, expectedEq1);
+ Expression actualOr = SparkV2Filters.convert(or);
+
Assertions.assertThat(actualOr.toString()).isEqualTo(expectedOr.toString());
+
+ Predicate orWithInvalidLeft = new Or(invalid, eq1);
+ Expression convertedOr = SparkV2Filters.convert(orWithInvalidLeft);
+ Assertions.assertThat(convertedOr).isNull();
+
+ Predicate not = new Not(lt1);
+ Expression expectedNot = Expressions.not(expectedLt1);
+ Expression actualNot = SparkV2Filters.convert(not);
+
Assertions.assertThat(actualNot.toString()).isEqualTo(expectedNot.toString());
+ }
+
+ private org.apache.spark.sql.connector.expressions.Expression[] expressions(
+ org.apache.spark.sql.connector.expressions.Expression... expressions) {
+ return expressions;
+ }
+
+ private static int dateToYears(String dateString) {
Review Comment:
Why are there so many conversion functions here? There are conversion
functions in the new helper as well. Are they all used?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
