This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new fa14834450 [common] Keep the selected variant schema when its evidence
is gone (#9532)
fa14834450 is described below
commit fa14834450056850d2f01a0ba80be9b3cd188a56
Author: ZIHAN DAI <[email protected]>
AuthorDate: Sun Sep 20 15:09:52 2026 +1000
[common] Keep the selected variant schema when its evidence is gone (#9532)
---
.../data/variant/InferVariantShreddingSchema.java | 49 ++++++-
.../variant/InferVariantShreddingSchemaTest.java | 143 +++++++++++++++++++++
2 files changed, 191 insertions(+), 1 deletion(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java
b/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java
index cf4d52e9d5..6f9f656d25 100644
---
a/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java
+++
b/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java
@@ -587,7 +587,11 @@ public class InferVariantShreddingSchema {
if (current != null && !(current instanceof VariantType)) {
combined = current;
} else if (previousSelected != null) {
- combined = previousSelected;
+ // This node has no evidence in the current file. A previously
selected schema
+ // is not evidence - its fields carry no counts - so it cannot
be run through
+ // admission and retention again; carry it forward as it is,
debiting the shared
+ // width budget for what it holds.
+ return retainSelectedSchema(previousSelected, maxFields);
} else {
return DataTypes.VARIANT();
}
@@ -667,6 +671,49 @@ public class InferVariantShreddingSchema {
return selectScalarType(combined, current, previousSelected);
}
+ /**
+ * Carries a previously selected schema forward for a node the current
file has no evidence for.
+ * The selection is already final, so nothing is re-thresholded, but its
nodes still consume the
+ * shared width budget. The entry unit for this node has already been
spent by the caller, so
+ * this mirrors what finalizeAdaptiveSchema does from that point on: a
child is entered only
+ * while budget remains, entering it spends one unit, and a child that
exhausts the budget
+ * becomes VARIANT while its field or array container is still kept.
+ */
+ private DataType retainSelectedSchema(DataType selected, MaxFields
maxFields) {
+ if (selected instanceof RowType) {
+ List<DataField> fields = new ArrayList<>();
+ for (DataField field : ((RowType) selected).getFields()) {
+ if (maxFields.remaining <= 0) {
+ break;
+ }
+ maxFields.remaining--;
+ DataType retained =
+ maxFields.remaining <= 0
+ ? DataTypes.VARIANT()
+ : retainSelectedSchema(field.type(),
maxFields);
+ fields.add(new DataField(fields.size(), field.name(),
retained));
+ }
+ return fields.isEmpty() ? DataTypes.VARIANT() : new
RowType(fields);
+ }
+ if (selected instanceof ArrayType) {
+ maxFields.remaining--;
+ DataType element =
+ maxFields.remaining <= 0
+ ? DataTypes.VARIANT()
+ : retainSelectedSchema(
+ ((ArrayType) selected).getElementType(),
maxFields);
+ return new ArrayType(element);
+ }
+ if (selected instanceof VariantType) {
+ // The caller has already spent this node's entry unit, and a
VARIANT has no typed child
+ // to spend another on - the evidence-driven walk also returns
VARIANT after the entry
+ // debit alone.
+ return selected;
+ }
+ maxFields.remaining--;
+ return selected;
+ }
+
private DataType selectScalarType(
DataType combined, DataType current, DataType previousSelected) {
if (current == null) {
diff --git
a/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java
b/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java
index 51acc76f16..346a6a194c 100644
---
a/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java
@@ -422,6 +422,149 @@ public class InferVariantShreddingSchemaTest {
new String[] {"historical"})));
}
+ /**
+ * A node can drift from scalar to object, which degrades its combined
evidence to VARIANT while
+ * the selected schema for it is a ROW, and then be absent from the next
file. With no evidence
+ * to fall back on, the selected schema is the only thing left - but it is
not evidence: its
+ * fields carry no counts, so it cannot be run through admission and
retention a second time.
+ */
+ @Test
+ void
testAdaptiveInferenceKeepsSelectedRowWhenEvidenceDegradedAndNodeIsAbsent() {
+ RowType schema = RowType.of(new DataType[] {DataTypes.VARIANT()}, new
String[] {"v"});
+ VariantShreddingInferenceSession session =
+ new VariantShreddingInferenceSession(
+ new InferVariantShreddingSchema(schema, 300, 50, 0.1),
256, 0.1, 0.05);
+
+ session.inferSchema(
+ Collections.singletonList(
+
GenericRow.of(GenericVariant.fromJson("{\"k\":1,\"p\":5}"))));
+ session.commitPendingInference();
+ session.inferSchema(
+ Collections.singletonList(
+
GenericRow.of(GenericVariant.fromJson("{\"k\":1,\"p\":{\"x\":1}}"))));
+ session.commitPendingInference();
+
+ RowType afterAbsence =
+ session.inferSchema(
+ Collections.singletonList(
+
GenericRow.of(GenericVariant.fromJson("{\"k\":1}"))));
+
+ assertThat(afterAbsence.getField("v").type())
+ .isEqualTo(
+ variantShreddingSchema(
+ RowType.of(
+ new DataType[] {
+ DataTypes.BIGINT(),
+ RowType.of(
+ new DataType[]
{DataTypes.BIGINT()},
+ new String[] {"x"})
+ },
+ new String[] {"k", "p"})));
+ }
+
+ /**
+ * maxSchemaWidth is one budget shared by every variant column. A schema
carried forward for a
+ * node with no evidence still occupies it, so a later column must not get
to spend what the
+ * carried-forward schema is holding.
+ */
+ @Test
+ void testRetainedSchemaStillConsumesTheSharedWidthBudget() {
+ RowType schema =
+ RowType.of(
+ new DataType[] {DataTypes.VARIANT(),
DataTypes.VARIANT()},
+ new String[] {"a", "b"});
+ VariantShreddingInferenceSession session =
+ new VariantShreddingInferenceSession(
+ new InferVariantShreddingSchema(schema, 8, 50, 0.1),
256, 0.1, 0.05);
+
+ session.inferSchema(
+ Collections.singletonList(
+ GenericRow.of(
+ GenericVariant.fromJson("{\"p\":5}"),
+ GenericVariant.fromJson("{\"q\":1}"))));
+ session.commitPendingInference();
+ session.inferSchema(
+ Collections.singletonList(
+ GenericRow.of(
+ GenericVariant.fromJson("{\"p\":{\"x\":1}}"),
+ GenericVariant.fromJson("{\"q\":1}"))));
+ session.commitPendingInference();
+
+ RowType afterAbsence =
+ session.inferSchema(
+ Collections.singletonList(
+ GenericRow.of(
+ GenericVariant.fromJson("{}"),
+
GenericVariant.fromJson("{\"q\":1,\"r\":1}"))));
+
+ // "a" keeps ROW<x BIGINT> under "p", and the budget it holds leaves
"r" untyped in "b".
+ assertThat(afterAbsence.getField("b").type().toString())
+ .contains("`q` ROW<`value` BYTES, `typed_value` BIGINT>")
+ .doesNotContain("`r` ROW<`value` BYTES, `typed_value`");
+ }
+
+ /**
+ * At the last budget unit the evidence-driven walk still keeps the field
and downgrades its
+ * child to VARIANT. Retaining a schema has to do the same rather than
drop the field, or the
+ * whole retained node collapses.
+ */
+ @Test
+ void testRetainedSchemaKeepsItsFieldsAtTheLastBudgetUnit() {
+ RowType schema =
+ RowType.of(
+ new DataType[] {DataTypes.VARIANT(),
DataTypes.VARIANT()},
+ new String[] {"a", "b"});
+ VariantShreddingInferenceSession session =
+ new VariantShreddingInferenceSession(
+ new InferVariantShreddingSchema(schema, 7, 50, 0.1),
256, 0.1, 0.05);
+
+ session.inferSchema(
+ Collections.singletonList(
+ GenericRow.of(GenericVariant.fromJson("1"),
GenericVariant.fromJson("5"))));
+ session.commitPendingInference();
+ session.inferSchema(
+ Collections.singletonList(
+ GenericRow.of(
+ GenericVariant.fromJson("1"),
+ GenericVariant.fromJson("{\"q\":1}"))));
+ session.commitPendingInference();
+
+ RowType afterAbsence =
+ session.inferSchema(
+ Collections.singletonList(
+
GenericRow.of(GenericVariant.fromJson("{\"x\":1,\"y\":1}"), null)));
+
+
assertThat(afterAbsence.getField("b").type().toString()).contains("`q`");
+ }
+
+ /**
+ * A retained VARIANT leaf costs the entry unit the caller already spent
and nothing more - it
+ * has no typed child to spend a second on. Charging it twice takes width
away from the columns
+ * that follow.
+ */
+ @Test
+ void testRetainedVariantLeafIsNotChargedTwice() {
+ RowType schema =
+ RowType.of(
+ new DataType[] {DataTypes.VARIANT(),
DataTypes.VARIANT()},
+ new String[] {"a", "b"});
+ VariantShreddingInferenceSession session =
+ new VariantShreddingInferenceSession(
+ new InferVariantShreddingSchema(schema, 3, 50, 0.1),
256, 0.1, 0.05);
+
+ session.inferSchema(
+ Collections.singletonList(GenericRow.of(null,
GenericVariant.fromJson("5"))));
+ session.commitPendingInference();
+
+ RowType afterAbsence =
+ session.inferSchema(
+ Collections.singletonList(
+ GenericRow.of(null,
GenericVariant.fromJson("6"))));
+
+ assertThat(afterAbsence.getField("b").type())
+ .isEqualTo(variantShreddingSchema(DataTypes.BIGINT()));
+ }
+
@Test
void testInferSchemaWithDeepNesting() {
// Schema: row<v: variant>
