JingsongLi commented on code in PR #11006:
URL: https://github.com/apache/paimon/pull/11006#discussion_r4493583004
##########
paimon-python/pypaimon/multimodal/columns.py:
##########
@@ -234,38 +250,56 @@ def add_columns(owner, schema, *, inputs, fn, batch_size):
if inputs is not None:
raise ValueError("inputs requires fn.")
else:
- if not callable(fn) or inputs is None:
- raise ValueError("A callable fn and explicit inputs are required
for backfill.")
- inputs = [inputs] if isinstance(inputs, str) else list(inputs)
- _validate_names(inputs, "inputs")
- missing = set(inputs).difference(raw.field_names)
- if missing:
- raise ValueError("Input columns do not exist: %s" %
sorted(missing))
+ inputs = _validate_inputs(raw, inputs, fn)
with _ColumnUpdate(owner, raw) as operation:
operation.add_schema(fields)
if fn is None:
return operation.finish(0)
- operation.phase = "compute"
- table = operation.raw.copy_without_time_travel({
- "blob-as-descriptor": "false", "read.batch-size": str(batch_size),
- })._copy_with_snapshot(operation.snapshot)
- builder = table.new_batch_write_builder()
- commit = operation.keep(builder.new_commit())
- update = builder.new_update().with_update_type(schema.names)
- update.with_read_projection(inputs)
- shard = operation.keep(update.new_shard_updator(0, 1))
- updated_rows = 0
- for batch in shard.arrow_reader():
- for start in range(0, batch.num_rows, batch_size):
- operation.phase = "compute"
- piece = batch.slice(start, batch_size)
- output = _normalize_output(fn(piece), pa.schema(fields))
- if output.num_rows != piece.num_rows:
- raise ValueError("Column function must return one output
row per input row, in input order.")
- operation.phase = "write"
- for output_batch in
output.to_batches(max_chunksize=batch_size):
- shard.update_by_arrow_batch(output_batch)
- updated_rows += output.num_rows
- operation.phase = "write"
- messages = shard.prepare_commit()
- return operation.finish(updated_rows, commit=commit, messages=messages)
+ return _compute_columns(operation, pa.schema(fields), inputs, fn,
batch_size)
+
+
+def compute_columns(owner, columns, *, inputs, fn, batch_size):
+ columns = [columns] if isinstance(columns, str) else list(columns)
+ _validate_names(columns, "Output")
+ _validate_batch_size(batch_size)
+ raw = owner.raw_table
+ _validate_update(raw)
+ missing = set(columns).difference(raw.field_names)
+ if missing:
+ raise ValueError("Output columns do not exist: %s" % sorted(missing))
+ if set(columns).intersection(raw.partition_keys):
+ raise ValueError("compute_columns cannot update partition columns.")
+ if set(columns).intersection(raw.options.video_frame_fields()):
+ raise ValueError("Use replace_video() to update video-frame columns.")
+ inputs = _validate_inputs(raw, inputs, fn)
+ target_schema = _target_schema(raw)
+ schema = pa.schema([target_schema.field(name) for name in columns])
Review Comment:
Rechecked 9488a4e: the original one-row Python cases now pass, as do 99
Python enrichment/shard tests. However, the valid nullable-parent case is still
rejected. The current [Native
CI](https://github.com/apache/paimon/actions/runs/42209669560/job/127868502114)
fails both new struct cases in
`test_compute_columns_restores_nested_nullability` at the final
`array.cast(field.type)` (2 failed, 25,474 passed).
A nullable struct may contain hidden child NULLs under a NULL parent even
when its child is declared NOT NULL. I also reproduced this through a real
Python-backed table using a fully validated `StructArray` representing `[{"x":
2}, None]`; `compute_columns` raises `ArrowInvalid: nulls in input cannot be
viewed as non-nullable`. Skipping casts when the array already has the
requested type makes the update and readback succeed, while a visible `{"x":
None}` remains rejected. Please avoid the redundant same-type casts (including
the newly added canonicalization cast for already-canonical outputs), so the
read/recompute identity path also works with Native reads and NULL parents.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]