stevenzwu commented on code in PR #14462: URL: https://github.com/apache/iceberg/pull/14462#discussion_r2529286265
########## api/src/test/java/org/apache/iceberg/expressions/TestIDReference.java: ########## @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.expressions; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.apache.iceberg.Schema; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; + +public class TestIDReference { + private static final Schema SCHEMA = + new Schema( + Types.NestedField.optional(34, "a", Types.IntegerType.get()), + Types.NestedField.required(35, "s", Types.StringType.get())); + + @Test + public void testIDReferenceEquality() { + IDReference<Integer> ref1 = new IDReference<>("a", 34); + IDReference<Integer> ref2 = new IDReference<>("a", 34); + IDReference<Integer> ref3 = new IDReference<>("b", 34); + IDReference<Integer> ref4 = new IDReference<>("a", 35); + + // Equal references + assertThat(ref1.id()).isEqualTo(ref2.id()); + assertThat(ref1.name()).isEqualTo(ref2.name()); + + // Different names, same fieldId + assertThat(ref1.id()).isEqualTo(ref3.id()); + assertThat(ref1.name()).isNotEqualTo(ref3.name()); + + // Same name, different fieldId + assertThat(ref1).isNotEqualTo(ref4); + } + + @Test + public void testIDReferenceBind() { + IDReference<Integer> ref = new IDReference<>("a", 34); + BoundReference<Integer> bound = ref.bind(SCHEMA.asStruct(), true); + + assertThat(bound).isInstanceOf(BoundReference.class); + assertThat(bound.fieldId()).isEqualTo(34); + assertThat(bound.name()).isEqualTo("a"); + assertThat(bound.type()).isEqualTo(Types.IntegerType.get()); + } + + @Test + public void testIDReferenceBindIgnoresCaseSensitivity() { + IDReference<Integer> ref = new IDReference<>("A", 34); + + // Should work regardless of case sensitivity since we use fieldId + BoundReference<Integer> bound1 = ref.bind(SCHEMA.asStruct(), true); + BoundReference<Integer> bound2 = ref.bind(SCHEMA.asStruct(), false); + + assertThat(bound1).isInstanceOf(BoundReference.class); + assertThat(bound2).isInstanceOf(BoundReference.class); + assertThat(bound1.fieldId()).isEqualTo(34); + assertThat(bound2.fieldId()).isEqualTo(34); + } + + @Test + public void testIDReferenceBindWithInvalidId() { + IDReference<Integer> ref = new IDReference<>("invalid", 999); + + assertThatThrownBy(() -> ref.bind(SCHEMA.asStruct(), true)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining( + "Cannot find field by id 999 in struct: struct<34: a: optional int, 35: s: required string>"); + } + + @Test + public void testIDReferenceRef() { + IDReference<Integer> ref = new IDReference<>("a", 34); + NamedReference<?> namedRef = ref.ref(); + + assertThat(namedRef.name()).isEqualTo("a"); + } + + @Test + public void testResolvedReferenceToString() { + IDReference<Integer> ref = new IDReference<>("a", 34); + + assertThat(ref.toString()).isEqualTo("ref(name=\"a\", id=\"34\")"); + } + + @Test + public void testIDReferenceExpressionIntegration() { + // Test that IDReference works in expression predicates + UnboundPredicate<?> expr = Expressions.equal(Expressions.ref("a", 34), 5); + assertThat(expr).isInstanceOf(UnboundPredicate.class); + + assertThat(expr.term()).isInstanceOf(IDReference.class); + + IDReference<?> resolvedRef = (IDReference<?>) expr.term(); Review Comment: nit: resolvedRef -> idRef ########## api/src/test/java/org/apache/iceberg/expressions/TestIDReference.java: ########## @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.expressions; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.apache.iceberg.Schema; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; + +public class TestIDReference { + private static final Schema SCHEMA = + new Schema( + Types.NestedField.optional(34, "a", Types.IntegerType.get()), + Types.NestedField.required(35, "s", Types.StringType.get())); + + @Test + public void testIDReferenceEquality() { + IDReference<Integer> ref1 = new IDReference<>("a", 34); + IDReference<Integer> ref2 = new IDReference<>("a", 34); + IDReference<Integer> ref3 = new IDReference<>("b", 34); + IDReference<Integer> ref4 = new IDReference<>("a", 35); + + // Equal references + assertThat(ref1.id()).isEqualTo(ref2.id()); Review Comment: I don't quite understand the equal test here. this is asserting on the id field of the IDRefrence (not the reference object themself). Not sure about the value of such assertion. `IDReference` or `NamedReference` doesn't implement `equals` method. At line 51, there is a equal comparison on ref1 and ref4. It will be just identity comparison (not equality comparison). ########## core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java: ########## @@ -250,13 +295,32 @@ private void term(Term term) throws IOException { throw new UnsupportedOperationException("Cannot write unsupported term: " + term); } - private void transform(String transform, String name) throws IOException { + private void transform(String transform, String name, Integer fieldId) throws IOException { gen.writeStartObject(); gen.writeStringField(TYPE, TRANSFORM); gen.writeStringField(TRANSFORM, transform); - gen.writeStringField(TERM, name); + gen.writeFieldName(TERM); + if (fieldId != null) { + reference(name, fieldId); + } else { + gen.writeString(name); + } gen.writeEndObject(); } + + private void reference(String name, Integer fieldId) throws IOException { + if (fieldId != null) { + gen.writeStartObject(); + gen.writeStringField(TYPE, REFERENCE); + if (name != null) { Review Comment: reference should always have a name? ########## api/src/test/java/org/apache/iceberg/expressions/TestIDReference.java: ########## @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.expressions; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.apache.iceberg.Schema; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; + +public class TestIDReference { + private static final Schema SCHEMA = + new Schema( + Types.NestedField.optional(34, "a", Types.IntegerType.get()), + Types.NestedField.required(35, "s", Types.StringType.get())); + + @Test + public void testIDReferenceEquality() { + IDReference<Integer> ref1 = new IDReference<>("a", 34); + IDReference<Integer> ref2 = new IDReference<>("a", 34); + IDReference<Integer> ref3 = new IDReference<>("b", 34); + IDReference<Integer> ref4 = new IDReference<>("a", 35); + + // Equal references + assertThat(ref1.id()).isEqualTo(ref2.id()); + assertThat(ref1.name()).isEqualTo(ref2.name()); + + // Different names, same fieldId + assertThat(ref1.id()).isEqualTo(ref3.id()); + assertThat(ref1.name()).isNotEqualTo(ref3.name()); + + // Same name, different fieldId + assertThat(ref1).isNotEqualTo(ref4); + } + + @Test + public void testIDReferenceBind() { + IDReference<Integer> ref = new IDReference<>("a", 34); + BoundReference<Integer> bound = ref.bind(SCHEMA.asStruct(), true); + + assertThat(bound).isInstanceOf(BoundReference.class); + assertThat(bound.fieldId()).isEqualTo(34); + assertThat(bound.name()).isEqualTo("a"); + assertThat(bound.type()).isEqualTo(Types.IntegerType.get()); + } + + @Test + public void testIDReferenceBindIgnoresCaseSensitivity() { + IDReference<Integer> ref = new IDReference<>("A", 34); + + // Should work regardless of case sensitivity since we use fieldId + BoundReference<Integer> bound1 = ref.bind(SCHEMA.asStruct(), true); + BoundReference<Integer> bound2 = ref.bind(SCHEMA.asStruct(), false); + + assertThat(bound1).isInstanceOf(BoundReference.class); + assertThat(bound2).isInstanceOf(BoundReference.class); + assertThat(bound1.fieldId()).isEqualTo(34); + assertThat(bound2.fieldId()).isEqualTo(34); + } + + @Test + public void testIDReferenceBindWithInvalidId() { + IDReference<Integer> ref = new IDReference<>("invalid", 999); + + assertThatThrownBy(() -> ref.bind(SCHEMA.asStruct(), true)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining( + "Cannot find field by id 999 in struct: struct<34: a: optional int, 35: s: required string>"); + } + + @Test + public void testIDReferenceRef() { + IDReference<Integer> ref = new IDReference<>("a", 34); + NamedReference<?> namedRef = ref.ref(); + + assertThat(namedRef.name()).isEqualTo("a"); + } + + @Test + public void testResolvedReferenceToString() { + IDReference<Integer> ref = new IDReference<>("a", 34); + + assertThat(ref.toString()).isEqualTo("ref(name=\"a\", id=\"34\")"); + } + + @Test + public void testIDReferenceExpressionIntegration() { + // Test that IDReference works in expression predicates + UnboundPredicate<?> expr = Expressions.equal(Expressions.ref("a", 34), 5); + assertThat(expr).isInstanceOf(UnboundPredicate.class); + + assertThat(expr.term()).isInstanceOf(IDReference.class); + + IDReference<?> resolvedRef = (IDReference<?>) expr.term(); + assertThat(resolvedRef.name()).isEqualTo("a"); + assertThat(resolvedRef.id()).isEqualTo(34); + } + + @Test + public void testIDReferenceUnbind() { + // Test that unbinding a bound reference returns a NamedReference for compatibility + Expression expr = Expressions.equal(Expressions.ref("a", 34), 5); + Expression boundExpr = Binder.bind(SCHEMA.asStruct(), expr, true); + + assertThat(boundExpr).isInstanceOf(BoundPredicate.class); + BoundPredicate<?> boundPred = (BoundPredicate<?>) boundExpr; + + UnboundTerm<?> unbound = ExpressionUtil.unbind(boundPred.term()); + assertThat(unbound).isInstanceOf(NamedReference.class); Review Comment: We discussed this previously. Now with `IDReference` extends from `NamedReference`, we can change the `ExpressionUtil.unbind` implementation to always return an `IDReference` object without change the API return type (stay as `NamedReference`). ########## core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java: ########## @@ -250,13 +295,32 @@ private void term(Term term) throws IOException { throw new UnsupportedOperationException("Cannot write unsupported term: " + term); } - private void transform(String transform, String name) throws IOException { + private void transform(String transform, String name, Integer fieldId) throws IOException { gen.writeStartObject(); gen.writeStringField(TYPE, TRANSFORM); gen.writeStringField(TRANSFORM, transform); - gen.writeStringField(TERM, name); + gen.writeFieldName(TERM); + if (fieldId != null) { + reference(name, fieldId); + } else { Review Comment: this `else` case is also taken care by the `reference(...)` method below. seems redundant here ########## core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java: ########## @@ -236,11 +264,28 @@ private String operationType(Expression.Operation op) { private void term(Term term) throws IOException { if (term instanceof UnboundTransform) { UnboundTransform<?, ?> transform = (UnboundTransform<?, ?>) term; - transform(transform.transform().toString(), transform.ref().name()); + NamedReference<?> ref = transform.ref(); + Integer fieldId = null; + if (mode == SerializationMode.WITH_FIELD_IDS && ref instanceof IDReference) { Review Comment: I am wondering if we need the `SerializationMode` config. ########## api/src/test/java/org/apache/iceberg/expressions/TestIDReference.java: ########## @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.expressions; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.apache.iceberg.Schema; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; + +public class TestIDReference { + private static final Schema SCHEMA = + new Schema( + Types.NestedField.optional(34, "a", Types.IntegerType.get()), + Types.NestedField.required(35, "s", Types.StringType.get())); + + @Test + public void testIDReferenceEquality() { + IDReference<Integer> ref1 = new IDReference<>("a", 34); + IDReference<Integer> ref2 = new IDReference<>("a", 34); + IDReference<Integer> ref3 = new IDReference<>("b", 34); + IDReference<Integer> ref4 = new IDReference<>("a", 35); + + // Equal references + assertThat(ref1.id()).isEqualTo(ref2.id()); + assertThat(ref1.name()).isEqualTo(ref2.name()); + + // Different names, same fieldId + assertThat(ref1.id()).isEqualTo(ref3.id()); + assertThat(ref1.name()).isNotEqualTo(ref3.name()); + + // Same name, different fieldId + assertThat(ref1).isNotEqualTo(ref4); + } + + @Test + public void testIDReferenceBind() { + IDReference<Integer> ref = new IDReference<>("a", 34); + BoundReference<Integer> bound = ref.bind(SCHEMA.asStruct(), true); + + assertThat(bound).isInstanceOf(BoundReference.class); + assertThat(bound.fieldId()).isEqualTo(34); + assertThat(bound.name()).isEqualTo("a"); + assertThat(bound.type()).isEqualTo(Types.IntegerType.get()); + } + + @Test + public void testIDReferenceBindIgnoresCaseSensitivity() { + IDReference<Integer> ref = new IDReference<>("A", 34); + + // Should work regardless of case sensitivity since we use fieldId + BoundReference<Integer> bound1 = ref.bind(SCHEMA.asStruct(), true); + BoundReference<Integer> bound2 = ref.bind(SCHEMA.asStruct(), false); + + assertThat(bound1).isInstanceOf(BoundReference.class); + assertThat(bound2).isInstanceOf(BoundReference.class); + assertThat(bound1.fieldId()).isEqualTo(34); + assertThat(bound2.fieldId()).isEqualTo(34); + } + + @Test + public void testIDReferenceBindWithInvalidId() { + IDReference<Integer> ref = new IDReference<>("invalid", 999); + + assertThatThrownBy(() -> ref.bind(SCHEMA.asStruct(), true)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining( + "Cannot find field by id 999 in struct: struct<34: a: optional int, 35: s: required string>"); + } + + @Test + public void testIDReferenceRef() { + IDReference<Integer> ref = new IDReference<>("a", 34); + NamedReference<?> namedRef = ref.ref(); + + assertThat(namedRef.name()).isEqualTo("a"); + } + + @Test + public void testResolvedReferenceToString() { Review Comment: nit: method name should be `testIDReferenceToString` ########## core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java: ########## @@ -250,13 +295,32 @@ private void term(Term term) throws IOException { throw new UnsupportedOperationException("Cannot write unsupported term: " + term); } - private void transform(String transform, String name) throws IOException { + private void transform(String transform, String name, Integer fieldId) throws IOException { gen.writeStartObject(); gen.writeStringField(TYPE, TRANSFORM); gen.writeStringField(TRANSFORM, transform); - gen.writeStringField(TERM, name); + gen.writeFieldName(TERM); + if (fieldId != null) { + reference(name, fieldId); + } else { + gen.writeString(name); + } gen.writeEndObject(); } + + private void reference(String name, Integer fieldId) throws IOException { + if (fieldId != null) { + gen.writeStartObject(); + gen.writeStringField(TYPE, REFERENCE); + if (name != null) { + gen.writeStringField(TERM, name); Review Comment: outside field name is also called `TERM` at line 302. seems a bit weird to also use `TERM` here. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
