sshiv012 commented on code in PR #8769:
URL: https://github.com/apache/texera/pull/8769#discussion_r4223159712
##########
amber/src/main/python/core/runnables/data_processor.py:
##########
@@ -132,16 +141,40 @@ def _set_output_tuple(self, output_iterator:
Iterator[Optional[TupleLike]]) -> N
# output could be a None, a TupleLike, or a TableLike.
for output_tuple in all_output_to_tuple(output):
if output_tuple is not None:
- output_tuple.finalize(
- self._context.output_manager.get_port().get_schema()
- )
+ try:
+ output_tuple.finalize(
+
self._context.output_manager.get_port().get_schema()
+ )
+ except (KeyError, TypeError) as error:
+ # Only explain on failure, so valid rows are not
inspected.
+ raise self._explain_schema_mismatch(output, error)
from error
self._switch_context()
self._context.tuple_processing_manager.current_output_tuple = (
output_tuple
)
self._switch_context()
self._context.tuple_processing_manager.finished_current.set()
+ @staticmethod
+ def _explain_schema_mismatch(output, error: Exception) -> Exception:
+ """
+ Rebuild a schema check failure with a hint on how to fix the output.
+ A dict holding DataFrames is valid for BINARY columns (they are
+ pickled), so it is only called out once the schema check has failed.
+ """
+ if isinstance(output, dict) and any(
Review Comment:
Fixed. The DataFrame hint now only fires on a type error in a non-binary
column and keeps the original message, so a misspelled or missing column gets
the normal column hint. Added tests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
