JingsongLi commented on code in PR #11006:
URL: https://github.com/apache/paimon/pull/11006#discussion_r4490676055
##########
paimon-python/pypaimon/multimodal/columns.py:
##########
@@ -234,38 +250,56 @@ def add_columns(owner, schema, *, inputs, fn, batch_size):
if inputs is not None:
raise ValueError("inputs requires fn.")
else:
- if not callable(fn) or inputs is None:
- raise ValueError("A callable fn and explicit inputs are required
for backfill.")
- inputs = [inputs] if isinstance(inputs, str) else list(inputs)
- _validate_names(inputs, "inputs")
- missing = set(inputs).difference(raw.field_names)
- if missing:
- raise ValueError("Input columns do not exist: %s" %
sorted(missing))
+ inputs = _validate_inputs(raw, inputs, fn)
with _ColumnUpdate(owner, raw) as operation:
operation.add_schema(fields)
if fn is None:
return operation.finish(0)
- operation.phase = "compute"
- table = operation.raw.copy_without_time_travel({
- "blob-as-descriptor": "false", "read.batch-size": str(batch_size),
- })._copy_with_snapshot(operation.snapshot)
- builder = table.new_batch_write_builder()
- commit = operation.keep(builder.new_commit())
- update = builder.new_update().with_update_type(schema.names)
- update.with_read_projection(inputs)
- shard = operation.keep(update.new_shard_updator(0, 1))
- updated_rows = 0
- for batch in shard.arrow_reader():
- for start in range(0, batch.num_rows, batch_size):
- operation.phase = "compute"
- piece = batch.slice(start, batch_size)
- output = _normalize_output(fn(piece), pa.schema(fields))
- if output.num_rows != piece.num_rows:
- raise ValueError("Column function must return one output
row per input row, in input order.")
- operation.phase = "write"
- for output_batch in
output.to_batches(max_chunksize=batch_size):
- shard.update_by_arrow_batch(output_batch)
- updated_rows += output.num_rows
- operation.phase = "write"
- messages = shard.prepare_commit()
- return operation.finish(updated_rows, commit=commit, messages=messages)
+ return _compute_columns(operation, pa.schema(fields), inputs, fn,
batch_size)
+
+
+def compute_columns(owner, columns, *, inputs, fn, batch_size):
+ columns = [columns] if isinstance(columns, str) else list(columns)
+ _validate_names(columns, "Output")
+ _validate_batch_size(batch_size)
+ raw = owner.raw_table
+ _validate_update(raw)
+ missing = set(columns).difference(raw.field_names)
+ if missing:
+ raise ValueError("Output columns do not exist: %s" % sorted(missing))
+ if set(columns).intersection(raw.partition_keys):
+ raise ValueError("compute_columns cannot update partition columns.")
+ if set(columns).intersection(raw.options.video_frame_fields()):
+ raise ValueError("Use replace_video() to update video-frame columns.")
+ inputs = _validate_inputs(raw, inputs, fn)
+ target_schema = _target_schema(raw)
+ schema = pa.schema([target_schema.field(name) for name in columns])
Review Comment:
[P2] Normalize nested read nullability before validating computed outputs
For an existing `value` column of `pa.struct([pa.field('x', pa.int64(),
nullable=False)])`, `compute_columns(['value'], inputs=['value'], fn=lambda b:
b)` fails with `Incompatible type` even when the only value is `{'x': 2}`;
non-nullable ARRAY elements fail too. `read_arrow_schema` deliberately relaxes
nested nullable flags, while this target schema retains the declared
constraints, so the strict output check rejects the API's own valid read result
before the shard's write normalization. Please normalize nested nullability
against the target before strict type comparison, with coverage for valid
read/recompute outputs and rejection of actual child NULLs.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]