Yicong-Huang commented on code in PR #58073: URL: https://github.com/apache/spark/pull/58073#discussion_r3816631360
########## python/pyspark/tests/upstream/pyarrow/test_pyarrow_dataframe_from_pandas.py: ########## @@ -0,0 +1,172 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Golden-file tests for the PyArrow ``from_pandas`` constructors that take a whole pandas +DataFrame: ``pa.RecordBatch.from_pandas`` (with ``pa.Table.from_pandas`` / +``pa.Schema.from_pandas`` to follow in this file). These take a DataFrame, unlike +``pa.Array.from_pandas`` which takes a Series (covered by test_pyarrow_array_from_pandas_*). + +Per-column type inference matches the Array tests, so these pin the DataFrame-level +behavior instead: whole-frame assembly, the pandas index under ``preserve_index``, and -- +for RecordBatch -- num_rows preservation for a 0-column DataFrame. Spark calls +``RecordBatch.from_pandas`` bare at pandas/conversion.py:1026 and connect/session.py:632 +(the createDataFrame 0-column branch) and stateful_processor_api_client.py:557, relying on +the default ``preserve_index=None`` to carry num_rows via the index metadata -- otherwise a +0-column relation loses its rows. + +Regenerate with SPARK_GENERATE_GOLDEN_FILES=1. +""" + +import datetime +import unittest + +from pyspark.testing.utils import ( + have_pyarrow, + have_pandas, + pyarrow_requirement_message, + pandas_requirement_message, +) +from pyspark.testing.goldenutils import GoldenFileTestMixin + +if have_pandas: + import pandas as pd +if have_pyarrow: + import pyarrow as pa + + +class _PyArrowFromPandasFrameTestBase(GoldenFileTestMixin, unittest.TestCase): + """ + Shared machinery for the DataFrame-input ``from_pandas`` constructors (RecordBatch + here; Table/Schema as followups). Owns the source-frame inventory and defines no + ``test_*`` of its own. + """ + + def _build_source_frames(self): + """Named pandas DataFrames covering shape x index-kind, plus a dtype sample.""" + dt = datetime.datetime(2020, 1, 1, 5, 30) + named = pd.Index([100, 200, 300], name="idx") + unnamed = pd.Index([10, 20, 30]) + frames = {} + + # ===================================================================== + # 0-column frames -- only the index carries the row count + # ===================================================================== Review Comment: can we also test df with multi col but 0 row? ########## python/pyspark/testing/goldenutils.py: ########## @@ -371,17 +385,25 @@ def repr_arrow_table_value(cls, value: Any, max_len: int = 32) -> str: str "{col: [val1, val2, None], ...}@Table[name: type, ...]" """ - columns = [] - for name, column in zip(value.column_names, value.columns): - # Escape NULL bytes so the value can be safely stored in CSV files. - elements = [cls._scalar_str(scalar) for scalar in column] - columns.append(f"{name}: [" + ", ".join(elements) + "]") - v_str = "{" + ", ".join(columns) + "}" - if max_len > 0: - v_str = v_str[:max_len] - schema = ", ".join(f"{f.name}: {cls.repr_type(f.type)}" for f in value.schema) + v_str, schema = cls._repr_arrow_columns(value, max_len) return f"{v_str}@Table[{schema}]" + @classmethod + def repr_arrow_record_batch_value(cls, value: Any, max_len: int = 32) -> str: Review Comment: I see the value field is in `Any` type. maybe we should change this to `RecordBatch`? I see other repr methods have similar issue. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
