singhpk234 commented on code in PR #18776: URL: https://github.com/apache/iceberg/pull/18776#discussion_r4496290023
########## api/src/main/java/org/apache/iceberg/Constraint.java: ########## @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.io.Serializable; + +/** + * A constraint declares a property that a table's rows are expected to satisfy. + * + * <p>A constraint's definition is stored in table metadata and references the fields it applies to + * by field ID, so it keeps applying after columns are renamed or reordered. The concrete subtype is Review Comment: is the fieldID explanation required in this class definition ? ########## core/src/main/java/org/apache/iceberg/BaseConstraint.java: ########## @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.util.List; +import java.util.Objects; +import java.util.Set; +import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; + +abstract class BaseConstraint implements Constraint { + private final int id; + private final String name; + private final boolean enforced; + private final long timestampMillis; + + private BaseConstraint(int id, String name, boolean enforced, long timestampMillis) { + Preconditions.checkArgument( + name != null && !name.isEmpty(), "Invalid constraint name: null or empty"); + this.id = id; + this.name = name; + this.enforced = enforced; + this.timestampMillis = timestampMillis; + } + + static CheckConstraint check( + int id, String name, boolean enforced, long timestampMillis, String expressionJson) { Review Comment: string expression ?? can we have the expression obj ########## core/src/main/java/org/apache/iceberg/BaseConstraint.java: ########## @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.util.List; +import java.util.Objects; +import java.util.Set; +import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; + +abstract class BaseConstraint implements Constraint { + private final int id; + private final String name; + private final boolean enforced; + private final long timestampMillis; + + private BaseConstraint(int id, String name, boolean enforced, long timestampMillis) { + Preconditions.checkArgument( + name != null && !name.isEmpty(), "Invalid constraint name: null or empty"); + this.id = id; + this.name = name; + this.enforced = enforced; + this.timestampMillis = timestampMillis; + } + + static CheckConstraint check( + int id, String name, boolean enforced, long timestampMillis, String expressionJson) { + return new Check(id, name, enforced, timestampMillis, expressionJson); + } + + static UniqueConstraint unique( + int id, String name, boolean enforced, long timestampMillis, List<Integer> fieldIds) { + return new Unique(id, name, enforced, timestampMillis, fieldIds); + } + + static PrimaryKeyConstraint primaryKey( + int id, String name, boolean enforced, long timestampMillis, List<Integer> fieldIds) { + return new PrimaryKey(id, name, enforced, timestampMillis, fieldIds); + } + + @Override + public int id() { + return id; + } + + @Override + public String name() { + return name; + } + + @Override + public boolean enforced() { + return enforced; + } + + @Override + public long timestampMillis() { + return timestampMillis; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } else if (other == null || getClass() != other.getClass()) { + return false; + } + + BaseConstraint that = (BaseConstraint) other; + return id == that.id + && name.equals(that.name) + && enforced == that.enforced + && timestampMillis == that.timestampMillis + && Objects.equals(payload(), that.payload()); + } + + @Override + public int hashCode() { + return Objects.hash(id, type(), name, enforced, timestampMillis, payload()); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("id", id) + .add("type", type()) + .add("name", name) + .add("enforced", enforced) + .add("timestamp-ms", timestampMillis) + .add("payload", payload()) + .toString(); + } + + /** Returns the type-specific definition, used for equality. */ + abstract Object payload(); + + static class Check extends BaseConstraint implements CheckConstraint { + // kept as JSON until ExpressionParser supports ID references + private final String expressionJson; + + private Check( + int id, String name, boolean enforced, long timestampMillis, String expressionJson) { + super(id, name, enforced, timestampMillis); + Preconditions.checkArgument( + expressionJson != null, "Invalid check constraint %s: missing expression", name); + this.expressionJson = expressionJson; + } + + @Override + public ConstraintType type() { + return ConstraintType.CHECK; + } + + String expressionJson() { Review Comment: I think we should have expression obj here ########## api/src/main/java/org/apache/iceberg/Constraint.java: ########## @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.io.Serializable; + +/** + * A constraint declares a property that a table's rows are expected to satisfy. + * + * <p>A constraint's definition is stored in table metadata and references the fields it applies to Review Comment: can you link the TableMetadata () from the code here ########## core/src/main/java/org/apache/iceberg/ConstraintParser.java: ########## @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import java.util.List; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.util.JsonUtil; + +public class ConstraintParser { + private ConstraintParser() {} + + private static final String CONSTRAINT_ID = "constraint-id"; + private static final String TYPE = "type"; + private static final String NAME = "name"; + private static final String ENFORCED = "enforced"; + private static final String TIMESTAMP_MS = "timestamp-ms"; + private static final String EXPRESSION = "expression"; + private static final String FIELD_IDS = "field-ids"; + + public static String toJson(Constraint constraint) { + return toJson(constraint, true); + } + + public static String toJson(Constraint constraint, boolean pretty) { + return JsonUtil.generate(gen -> toJson(constraint, gen), pretty); + } + + static void toJson(Constraint constraint, JsonGenerator generator) throws IOException { + Preconditions.checkArgument(constraint != null, "Invalid constraint: null"); + + generator.writeStartObject(); + generator.writeNumberField(CONSTRAINT_ID, constraint.id()); + generator.writeStringField(TYPE, constraint.type().typeName()); + generator.writeStringField(NAME, constraint.name()); + generator.writeBooleanField(ENFORCED, constraint.enforced()); + generator.writeNumberField(TIMESTAMP_MS, constraint.timestampMillis()); + + if (constraint instanceof BaseConstraint.Check) { + generator.writeFieldName(EXPRESSION); + JsonUtil.mapper() + .writeTree( + generator, + JsonUtil.mapper().readTree(((BaseConstraint.Check) constraint).expressionJson())); + } else if (constraint instanceof KeyConstraint) { + JsonUtil.writeIntegerArray(FIELD_IDS, ((KeyConstraint) constraint).fieldIds(), generator); + } else { + throw new IllegalArgumentException("Cannot serialize constraint: " + constraint); + } + + generator.writeEndObject(); + } + + public static Constraint fromJson(String json) { + return JsonUtil.parse(json, ConstraintParser::fromJson); + } + + static Constraint fromJson(JsonNode node) { + Preconditions.checkArgument(node != null, "Cannot parse constraint from null object"); + Preconditions.checkArgument( + node.isObject(), "Cannot parse constraint from non-object: %s", node); + + int constraintId = JsonUtil.getInt(CONSTRAINT_ID, node); Review Comment: why int and not long ? what is snapshot id ? ########## core/src/main/java/org/apache/iceberg/BaseConstraint.java: ########## @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.util.List; +import java.util.Objects; +import java.util.Set; +import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; + +abstract class BaseConstraint implements Constraint { + private final int id; + private final String name; + private final boolean enforced; + private final long timestampMillis; + + private BaseConstraint(int id, String name, boolean enforced, long timestampMillis) { + Preconditions.checkArgument( + name != null && !name.isEmpty(), "Invalid constraint name: null or empty"); + this.id = id; + this.name = name; + this.enforced = enforced; + this.timestampMillis = timestampMillis; + } + + static CheckConstraint check( + int id, String name, boolean enforced, long timestampMillis, String expressionJson) { + return new Check(id, name, enforced, timestampMillis, expressionJson); + } + + static UniqueConstraint unique( + int id, String name, boolean enforced, long timestampMillis, List<Integer> fieldIds) { + return new Unique(id, name, enforced, timestampMillis, fieldIds); + } + + static PrimaryKeyConstraint primaryKey( + int id, String name, boolean enforced, long timestampMillis, List<Integer> fieldIds) { + return new PrimaryKey(id, name, enforced, timestampMillis, fieldIds); + } Review Comment: why not public ? ########## core/src/main/java/org/apache/iceberg/ConstraintParser.java: ########## @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import java.util.List; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.util.JsonUtil; + +public class ConstraintParser { + private ConstraintParser() {} + + private static final String CONSTRAINT_ID = "constraint-id"; + private static final String TYPE = "type"; + private static final String NAME = "name"; + private static final String ENFORCED = "enforced"; + private static final String TIMESTAMP_MS = "timestamp-ms"; + private static final String EXPRESSION = "expression"; + private static final String FIELD_IDS = "field-ids"; + + public static String toJson(Constraint constraint) { + return toJson(constraint, true); + } + + public static String toJson(Constraint constraint, boolean pretty) { + return JsonUtil.generate(gen -> toJson(constraint, gen), pretty); + } + + static void toJson(Constraint constraint, JsonGenerator generator) throws IOException { + Preconditions.checkArgument(constraint != null, "Invalid constraint: null"); + + generator.writeStartObject(); + generator.writeNumberField(CONSTRAINT_ID, constraint.id()); + generator.writeStringField(TYPE, constraint.type().typeName()); + generator.writeStringField(NAME, constraint.name()); + generator.writeBooleanField(ENFORCED, constraint.enforced()); + generator.writeNumberField(TIMESTAMP_MS, constraint.timestampMillis()); + + if (constraint instanceof BaseConstraint.Check) { + generator.writeFieldName(EXPRESSION); + JsonUtil.mapper() + .writeTree( + generator, + JsonUtil.mapper().readTree(((BaseConstraint.Check) constraint).expressionJson())); Review Comment: i think this is mixing the parser responsibility with the constraint core class, i would not recommend doing that ########## api/src/main/java/org/apache/iceberg/Constraint.java: ########## @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.io.Serializable; + +/** + * A constraint declares a property that a table's rows are expected to satisfy. + * + * <p>A constraint's definition is stored in table metadata and references the fields it applies to + * by field ID, so it keeps applying after columns are renamed or reordered. The concrete subtype is + * determined by {@link #type()}: a {@link CheckConstraint} carries a predicate, and a {@link + * KeyConstraint} ({@link UniqueConstraint} or {@link PrimaryKeyConstraint}) carries field IDs. + * Iceberg does not evaluate constraints; enforcement and validation are done by engines. + * + * <p>Constraints are supported in format v4 and later. + */ +public interface Constraint extends Serializable { + /** Returns the ID of the constraint, assigned by Iceberg and unique within the table. */ + int id(); + + /** + * Returns the name of the constraint, unique within the table. + * + * <p>Names are for humans and must not be used to identify a constraint in metadata. + */ + String name(); + + /** Returns the constraint {@link ConstraintType type}. */ + ConstraintType type(); + + /** + * Returns whether writers must verify that the rows they add or modify satisfy this constraint. + * + * <p>When enforced, a writer that cannot verify the constraint must reject writes rather than add + * unverified rows. Iceberg core does not enforce constraints. + */ + boolean enforced(); + + /** + * Returns the time the constraint was created or last modified, in milliseconds since the Unix + * epoch. + * + * <p>This value is informational and must not be used to decide whether a constraint applies to a + * snapshot or whether it holds. Review Comment: may be too late to this party why is this required again ... is this just from audit pov ? ########## api/src/main/java/org/apache/iceberg/Constraint.java: ########## @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.io.Serializable; + +/** + * A constraint declares a property that a table's rows are expected to satisfy. + * + * <p>A constraint's definition is stored in table metadata and references the fields it applies to + * by field ID, so it keeps applying after columns are renamed or reordered. The concrete subtype is + * determined by {@link #type()}: a {@link CheckConstraint} carries a predicate, and a {@link + * KeyConstraint} ({@link UniqueConstraint} or {@link PrimaryKeyConstraint}) carries field IDs. + * Iceberg does not evaluate constraints; enforcement and validation are done by engines. Review Comment: engine is too abstract, we do expect this be enforced in writes such as pyiceberg right ? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
