jorisvandenbossche opened a new issue, #34880:
URL: https://github.com/apache/arrow/issues/34880

   See eg 
https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/builds/46696219. 
It seems to be related to int32 vs int64 being created
   
   <details>
   
   ```
   ================================== FAILURES 
===================================
   _____________ TestZeroCopyConversion.test_zero_copy_dictionaries 
______________
   self = <pyarrow.tests.test_pandas.TestZeroCopyConversion object at 
0x000001947D9150E0>
       def test_zero_copy_dictionaries(self):
           arr = pa.DictionaryArray.from_arrays(
               np.array([0, 0]),
               np.array([5]))
       
           result = arr.to_pandas(zero_copy_only=True)
           values = pd.Categorical([5, 5])
       
   >       tm.assert_series_equal(pd.Series(result), pd.Series(values),
                                  check_names=False)
   E       AssertionError: Attributes of Series are different
   E       
   E       Attribute "dtype" are different
   E       [left]:  CategoricalDtype(categories=[5], ordered=False)
   E       [right]: CategoricalDtype(categories=[5], ordered=False)
   pyarrow\tests\test_pandas.py:2578: AssertionError
   _______ test_dataset_read_pandas_common_metadata[_metadata-False-True] 
________
   tempdir = 
WindowsPath('C:/Users/appveyor/AppData/Local/Temp/1/pytest-of-appveyor/pytest-0/test_dataset_read_pandas_commo2')
   use_legacy_dataset = True, preserve_index = False, metadata_fname = 
'_metadata'
       @pytest.mark.pandas
       @parametrize_legacy_dataset
       @pytest.mark.parametrize('preserve_index', [True, False, None])
       @pytest.mark.parametrize('metadata_fname', ["_metadata", 
"_common_metadata"])
       def test_dataset_read_pandas_common_metadata(
           tempdir, use_legacy_dataset, preserve_index, metadata_fname
       ):
           # ARROW-1103
           nfiles = 5
           size = 5
       
           dirpath = tempdir / guid()
           dirpath.mkdir()
       
           test_data = []
           frames = []
           paths = []
           for i in range(nfiles):
               df = _test_dataframe(size, seed=i)
               df.index = pd.Index(np.arange(i * size, (i + 1) * size), 
name='index')
       
               path = dirpath / '{}.parquet'.format(i)
       
               table = pa.Table.from_pandas(df, preserve_index=preserve_index)
       
               # Obliterate metadata
               table = table.replace_schema_metadata(None)
               assert table.schema.metadata is None
       
               _write_table(table, path)
               test_data.append(table)
               frames.append(df)
               paths.append(path)
       
           # Write _metadata common file
           table_for_metadata = pa.Table.from_pandas(
               df, preserve_index=preserve_index
           )
           pq.write_metadata(table_for_metadata.schema, dirpath / 
metadata_fname)
       
           dataset = pq.ParquetDataset(dirpath, 
use_legacy_dataset=use_legacy_dataset)
           columns = ['uint8', 'strings']
           result = dataset.read_pandas(columns=columns).to_pandas()
           expected = pd.concat([x[columns] for x in frames])
           expected.index.name = (
               df.index.name if preserve_index is not False else None)
   >       tm.assert_frame_equal(result, expected)
   pyarrow\tests\parquet\test_pandas.py:698: 
   _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
_ _
   left = RangeIndex(start=0, stop=25, step=1)
   right = Index([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 
15, 16, 17,
          18, 19, 20, 21, 22, 23, 24],
         dtype='int32')
   obj = 'DataFrame.index'
       def _check_types(left, right, obj: str = "Index") -> None:
           if not exact:
               return
       
           assert_class_equal(left, right, exact=exact, obj=obj)
           assert_attr_equal("inferred_type", left, right, obj=obj)
       
           # Skip exact dtype checking when `check_categorical` is False
           if is_categorical_dtype(left.dtype) and 
is_categorical_dtype(right.dtype):
               if check_categorical:
                   assert_attr_equal("dtype", left, right, obj=obj)
                   assert_index_equal(left.categories, right.categories, 
exact=exact)
               return
       
   >       assert_attr_equal("dtype", left, right, obj=obj)
   E       AssertionError: DataFrame.index are different
   E       
   E       Attribute "dtype" are different
   E       [left]:  int64
   E       [right]: int32
   
C:\Miniconda38-x64\envs\arrow\lib\site-packages\pandas\_testing\asserters.py:247:
 AssertionError
   _______ test_dataset_read_pandas_common_metadata[_metadata-False-False] 
_______
   tempdir = 
WindowsPath('C:/Users/appveyor/AppData/Local/Temp/1/pytest-of-appveyor/pytest-0/test_dataset_read_pandas_commo3')
   use_legacy_dataset = False, preserve_index = False, metadata_fname = 
'_metadata'
       @pytest.mark.pandas
       @parametrize_legacy_dataset
       @pytest.mark.parametrize('preserve_index', [True, False, None])
       @pytest.mark.parametrize('metadata_fname', ["_metadata", 
"_common_metadata"])
       def test_dataset_read_pandas_common_metadata(
           tempdir, use_legacy_dataset, preserve_index, metadata_fname
       ):
           # ARROW-1103
           nfiles = 5
           size = 5
       
           dirpath = tempdir / guid()
           dirpath.mkdir()
       
           test_data = []
           frames = []
           paths = []
           for i in range(nfiles):
               df = _test_dataframe(size, seed=i)
               df.index = pd.Index(np.arange(i * size, (i + 1) * size), 
name='index')
       
               path = dirpath / '{}.parquet'.format(i)
       
               table = pa.Table.from_pandas(df, preserve_index=preserve_index)
       
               # Obliterate metadata
               table = table.replace_schema_metadata(None)
               assert table.schema.metadata is None
       
               _write_table(table, path)
               test_data.append(table)
               frames.append(df)
               paths.append(path)
       
           # Write _metadata common file
           table_for_metadata = pa.Table.from_pandas(
               df, preserve_index=preserve_index
           )
           pq.write_metadata(table_for_metadata.schema, dirpath / 
metadata_fname)
       
           dataset = pq.ParquetDataset(dirpath, 
use_legacy_dataset=use_legacy_dataset)
           columns = ['uint8', 'strings']
           result = dataset.read_pandas(columns=columns).to_pandas()
           expected = pd.concat([x[columns] for x in frames])
           expected.index.name = (
               df.index.name if preserve_index is not False else None)
   >       tm.assert_frame_equal(result, expected)
   pyarrow\tests\parquet\test_pandas.py:698: 
   _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
_ _
   left = RangeIndex(start=0, stop=25, step=1)
   right = Index([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 
15, 16, 17,
          18, 19, 20, 21, 22, 23, 24],
         dtype='int32')
   obj = 'DataFrame.index'
       def _check_types(left, right, obj: str = "Index") -> None:
           if not exact:
               return
       
           assert_class_equal(left, right, exact=exact, obj=obj)
           assert_attr_equal("inferred_type", left, right, obj=obj)
       
           # Skip exact dtype checking when `check_categorical` is False
           if is_categorical_dtype(left.dtype) and 
is_categorical_dtype(right.dtype):
               if check_categorical:
                   assert_attr_equal("dtype", left, right, obj=obj)
                   assert_index_equal(left.categories, right.categories, 
exact=exact)
               return
       
   >       assert_attr_equal("dtype", left, right, obj=obj)
   E       AssertionError: DataFrame.index are different
   E       
   E       Attribute "dtype" are different
   E       [left]:  int64
   E       [right]: int32
   
C:\Miniconda38-x64\envs\arrow\lib\site-packages\pandas\_testing\asserters.py:247:
 AssertionError
   ____ test_dataset_read_pandas_common_metadata[_common_metadata-False-True] 
____
   tempdir = 
WindowsPath('C:/Users/appveyor/AppData/Local/Temp/1/pytest-of-appveyor/pytest-0/test_dataset_read_pandas_commo8')
   use_legacy_dataset = True, preserve_index = False
   metadata_fname = '_common_metadata'
       @pytest.mark.pandas
       @parametrize_legacy_dataset
       @pytest.mark.parametrize('preserve_index', [True, False, None])
       @pytest.mark.parametrize('metadata_fname', ["_metadata", 
"_common_metadata"])
       def test_dataset_read_pandas_common_metadata(
           tempdir, use_legacy_dataset, preserve_index, metadata_fname
       ):
           # ARROW-1103
           nfiles = 5
           size = 5
       
           dirpath = tempdir / guid()
           dirpath.mkdir()
       
           test_data = []
           frames = []
           paths = []
           for i in range(nfiles):
               df = _test_dataframe(size, seed=i)
               df.index = pd.Index(np.arange(i * size, (i + 1) * size), 
name='index')
       
               path = dirpath / '{}.parquet'.format(i)
       
               table = pa.Table.from_pandas(df, preserve_index=preserve_index)
       
               # Obliterate metadata
               table = table.replace_schema_metadata(None)
               assert table.schema.metadata is None
       
               _write_table(table, path)
               test_data.append(table)
               frames.append(df)
               paths.append(path)
       
           # Write _metadata common file
           table_for_metadata = pa.Table.from_pandas(
               df, preserve_index=preserve_index
           )
           pq.write_metadata(table_for_metadata.schema, dirpath / 
metadata_fname)
       
           dataset = pq.ParquetDataset(dirpath, 
use_legacy_dataset=use_legacy_dataset)
           columns = ['uint8', 'strings']
           result = dataset.read_pandas(columns=columns).to_pandas()
           expected = pd.concat([x[columns] for x in frames])
           expected.index.name = (
               df.index.name if preserve_index is not False else None)
   >       tm.assert_frame_equal(result, expected)
   pyarrow\tests\parquet\test_pandas.py:698: 
   _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
_ _
   left = RangeIndex(start=0, stop=25, step=1)
   right = Index([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 
15, 16, 17,
          18, 19, 20, 21, 22, 23, 24],
         dtype='int32')
   obj = 'DataFrame.index'
       def _check_types(left, right, obj: str = "Index") -> None:
           if not exact:
               return
       
           assert_class_equal(left, right, exact=exact, obj=obj)
           assert_attr_equal("inferred_type", left, right, obj=obj)
       
           # Skip exact dtype checking when `check_categorical` is False
           if is_categorical_dtype(left.dtype) and 
is_categorical_dtype(right.dtype):
               if check_categorical:
                   assert_attr_equal("dtype", left, right, obj=obj)
                   assert_index_equal(left.categories, right.categories, 
exact=exact)
               return
       
   >       assert_attr_equal("dtype", left, right, obj=obj)
   E       AssertionError: DataFrame.index are different
   E       
   E       Attribute "dtype" are different
   E       [left]:  int64
   E       [right]: int32
   
C:\Miniconda38-x64\envs\arrow\lib\site-packages\pandas\_testing\asserters.py:247:
 AssertionError
   ___ test_dataset_read_pandas_common_metadata[_common_metadata-False-False] 
____
   tempdir = 
WindowsPath('C:/Users/appveyor/AppData/Local/Temp/1/pytest-of-appveyor/pytest-0/test_dataset_read_pandas_commo9')
   use_legacy_dataset = False, preserve_index = False
   metadata_fname = '_common_metadata'
       @pytest.mark.pandas
       @parametrize_legacy_dataset
       @pytest.mark.parametrize('preserve_index', [True, False, None])
       @pytest.mark.parametrize('metadata_fname', ["_metadata", 
"_common_metadata"])
       def test_dataset_read_pandas_common_metadata(
           tempdir, use_legacy_dataset, preserve_index, metadata_fname
       ):
           # ARROW-1103
           nfiles = 5
           size = 5
       
           dirpath = tempdir / guid()
           dirpath.mkdir()
       
           test_data = []
           frames = []
           paths = []
           for i in range(nfiles):
               df = _test_dataframe(size, seed=i)
               df.index = pd.Index(np.arange(i * size, (i + 1) * size), 
name='index')
       
               path = dirpath / '{}.parquet'.format(i)
       
               table = pa.Table.from_pandas(df, preserve_index=preserve_index)
       
               # Obliterate metadata
               table = table.replace_schema_metadata(None)
               assert table.schema.metadata is None
       
               _write_table(table, path)
               test_data.append(table)
               frames.append(df)
               paths.append(path)
       
           # Write _metadata common file
           table_for_metadata = pa.Table.from_pandas(
               df, preserve_index=preserve_index
           )
           pq.write_metadata(table_for_metadata.schema, dirpath / 
metadata_fname)
       
           dataset = pq.ParquetDataset(dirpath, 
use_legacy_dataset=use_legacy_dataset)
           columns = ['uint8', 'strings']
           result = dataset.read_pandas(columns=columns).to_pandas()
           expected = pd.concat([x[columns] for x in frames])
           expected.index.name = (
               df.index.name if preserve_index is not False else None)
   >       tm.assert_frame_equal(result, expected)
   pyarrow\tests\parquet\test_pandas.py:698: 
   _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
_ _
   left = RangeIndex(start=0, stop=25, step=1)
   right = Index([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 
15, 16, 17,
          18, 19, 20, 21, 22, 23, 24],
         dtype='int32')
   obj = 'DataFrame.index'
       def _check_types(left, right, obj: str = "Index") -> None:
           if not exact:
               return
       
           assert_class_equal(left, right, exact=exact, obj=obj)
           assert_attr_equal("inferred_type", left, right, obj=obj)
       
           # Skip exact dtype checking when `check_categorical` is False
           if is_categorical_dtype(left.dtype) and 
is_categorical_dtype(right.dtype):
               if check_categorical:
                   assert_attr_equal("dtype", left, right, obj=obj)
                   assert_index_equal(left.categories, right.categories, 
exact=exact)
               return
       
   >       assert_attr_equal("dtype", left, right, obj=obj)
   E       AssertionError: DataFrame.index are different
   E       
   E       Attribute "dtype" are different
   E       [left]:  int64
   E       [right]: int32
   
C:\Miniconda38-x64\envs\arrow\lib\site-packages\pandas\_testing\asserters.py:247:
 AssertionError
   ```
   
   </details>


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to