felipecrv commented on PR #40416:
URL: https://github.com/apache/arrow/pull/40416#issuecomment-2003957845
I spent a lot of time trying to figure out what could possibly be causing
these regressions, but it turns out to be randomness caused by memory
operations and branch predictions.
When running the benchmarks on a commit that doesn't even touch the code I
get sizeable regressions too.
```
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Regressions: (42)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
benchmark baseline
contender change %
counters
FilterFSLInt64FilterNoNulls/4194304/8 12.655 GiB/sec 11.998
GiB/sec -5.188
{'family_index': 2, 'per_family_instance_index': 8, 'run_name':
'FilterFSLInt64FilterNoNulls/4194304/8', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 2257, 'data null%': 1.0, 'mask null%': 0.0,
'select%': 1.0}
FilterRecordBatchWithNulls/100/9 5.371 GiB/sec 5.078
GiB/sec -5.448 {'family_index': 7, 'per_family_instance_index': 39,
'run_name': 'FilterRecordBatchWithNulls/100/9', 'repetitions': 1,
'repetition_index': 0, 'threads': 1, 'iterations': 50, 'data null%': 10.0,
'extracted_size': 75952000.0, 'mask null%': 5.0, 'num_cols': 100.0, 'select%':
99.9}
FilterInt64FilterNoNulls/4194304/7 1.658 GiB/sec 1.566
GiB/sec -5.562
{'family_index': 0, 'per_family_instance_index': 7, 'run_name':
'FilterInt64FilterNoNulls/4194304/7', 'repetitions': 1, 'repetition_index': 0,
'threads': 1, 'iterations': 299, 'data null%': 1.0, 'mask null%': 0.0,
'select%': 50.0}
FilterRecordBatchWithNulls/50/7 3.781 GiB/sec 3.570
GiB/sec -5.596 {'family_index': 7, 'per_family_instance_index': 22,
'run_name': 'FilterRecordBatchWithNulls/50/7', 'repetitions': 1,
'repetition_index': 0, 'threads': 1, 'iterations': 68, 'data null%': 1.0,
'extracted_size': 37909600.0, 'mask null%': 5.0, 'num_cols': 50.0, 'select%':
50.0}
FilterRecordBatchWithNulls/100/10 3.966 GiB/sec 3.733
GiB/sec -5.883 {'family_index': 7, 'per_family_instance_index': 40,
'run_name': 'FilterRecordBatchWithNulls/100/10', 'repetitions': 1,
'repetition_index': 0, 'threads': 1, 'iterations': 69, 'data null%': 10.0,
'extracted_size': 37952800.0, 'mask null%': 5.0, 'num_cols': 100.0, 'select%':
50.0}
TakeInt64RandomIndicesWithNulls/4194304/10 109.672M items/sec 102.977M
items/sec -6.104
{'family_index': 9, 'per_family_instance_index': 1, 'run_name':
'TakeInt64RandomIndicesWithNulls/4194304/10', 'repetitions': 1,
'repetition_index': 0, 'threads': 1, 'iterations': 18, 'null_percent': 10.0}
TakeInt64RandomIndicesWithNulls/4194304/1000 134.146M items/sec 125.634M
items/sec -6.346
{'family_index': 9, 'per_family_instance_index': 0, 'run_name':
'TakeInt64RandomIndicesWithNulls/4194304/1000', 'repetitions': 1,
'repetition_index': 0, 'threads': 1, 'iterations': 22, 'null_percent': 0.1}
TakeInt64RandomIndicesNoNulls/4194304/10 142.865M items/sec 133.343M
items/sec -6.665
{'family_index': 8, 'per_family_instance_index': 1,
'run_name': 'TakeInt64RandomIndicesNoNulls/4194304/10', 'repetitions': 1,
'repetition_index': 0, 'threads': 1, 'iterations': 24, 'null_percent': 10.0}
FilterStringFilterNoNulls/4194304/7 1.261 GiB/sec 1.177
GiB/sec -6.730
{'family_index': 4, 'per_family_instance_index': 7, 'run_name':
'FilterStringFilterNoNulls/4194304/7', 'repetitions': 1, 'repetition_index': 0,
'threads': 1, 'iterations': 224, 'data null%': 1.0, 'mask null%': 0.0,
'select%': 50.0}
FilterRecordBatchWithNulls/50/6 6.255 GiB/sec 5.830
GiB/sec -6.799 {'family_index': 7, 'per_family_instance_index': 21,
'run_name': 'FilterRecordBatchWithNulls/50/6', 'repetitions': 1,
'repetition_index': 0, 'threads': 1, 'iterations': 58, 'data null%': 1.0,
'extracted_size': 75924000.0, 'mask null%': 5.0, 'num_cols': 50.0, 'select%':
99.9}
TakeInt64RandomIndicesNoNulls/4194304/1000 136.858M items/sec 127.408M
items/sec -6.906
{'family_index': 8, 'per_family_instance_index': 0, 'run_name':
'TakeInt64RandomIndicesNoNulls/4194304/1000', 'repetitions': 1,
'repetition_index': 0, 'threads': 1, 'iterations': 23, 'null_percent': 0.1}
FilterRecordBatchWithNulls/100/0 5.548 GiB/sec 5.157
GiB/sec -7.059 {'family_index': 7, 'per_family_instance_index': 30,
'run_name': 'FilterRecordBatchWithNulls/100/0', 'repetitions': 1,
'repetition_index': 0, 'threads': 1, 'iterations': 51, 'data null%': 0.0,
'extracted_size': 75952000.0, 'mask null%': 5.0, 'num_cols': 100.0, 'select%':
99.9}
FilterInt64FilterNoNulls/4194304/10 1.625 GiB/sec 1.505
GiB/sec -7.385
{'family_index': 0, 'per_family_instance_index': 10, 'run_name':
'FilterInt64FilterNoNulls/4194304/10', 'repetitions': 1, 'repetition_index': 0,
'threads': 1, 'iterations': 294, 'data null%': 10.0, 'mask null%': 0.0,
'select%': 50.0}
FilterRecordBatchWithNulls/100/12 5.554 GiB/sec 5.144
GiB/sec -7.385 {'family_index': 7, 'per_family_instance_index': 42,
'run_name': 'FilterRecordBatchWithNulls/100/12', 'repetitions': 1,
'repetition_index': 0, 'threads': 1, 'iterations': 52, 'data null%': 90.0,
'extracted_size': 75952000.0, 'mask null%': 5.0, 'num_cols': 100.0, 'select%':
99.9}
FilterStringFilterNoNulls/4194304/3 2.469 GiB/sec 2.285
GiB/sec -7.461
{'family_index': 4, 'per_family_instance_index': 3, 'run_name':
'FilterStringFilterNoNulls/4194304/3', 'repetitions': 1, 'repetition_index': 0,
'threads': 1, 'iterations': 446, 'data null%': 0.1, 'mask null%': 0.0,
'select%': 99.9}
FilterRecordBatchWithNulls/100/3 5.536 GiB/sec 5.100
GiB/sec -7.866 {'family_index': 7, 'per_family_instance_index': 33,
'run_name': 'FilterRecordBatchWithNulls/100/3', 'repetitions': 1,
'repetition_index': 0, 'threads': 1, 'iterations': 50, 'data null%': 0.1,
'extracted_size': 75952000.0, 'mask null%': 5.0, 'num_cols': 100.0, 'select%':
99.9}
FilterStringFilterNoNulls/4194304/0 2.847 GiB/sec 2.617
GiB/sec -8.056
{'family_index': 4, 'per_family_instance_index': 0, 'run_name':
'FilterStringFilterNoNulls/4194304/0', 'repetitions': 1, 'repetition_index': 0,
'threads': 1, 'iterations': 521, 'data null%': 0.0, 'mask null%': 0.0,
'select%': 99.9}
FilterInt64FilterNoNulls/4194304/13 1.636 GiB/sec 1.501
GiB/sec -8.242
{'family_index': 0, 'per_family_instance_index': 13, 'run_name':
'FilterInt64FilterNoNulls/4194304/13', 'repetitions': 1, 'repetition_index': 0,
'threads': 1, 'iterations': 294, 'data null%': 90.0, 'mask null%': 0.0,
'select%': 50.0}
FilterFSLInt64FilterNoNulls/4194304/4 1.101 GiB/sec 1.004
GiB/sec -8.853
{'family_index': 2, 'per_family_instance_index': 4, 'run_name':
'FilterFSLInt64FilterNoNulls/4194304/4', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 197, 'data null%': 0.1, 'mask null%': 0.0,
'select%': 50.0}
FilterRecordBatchWithNulls/100/7 3.947 GiB/sec 3.593
GiB/sec -8.964 {'family_index': 7, 'per_family_instance_index': 37,
'run_name': 'FilterRecordBatchWithNulls/100/7', 'repetitions': 1,
'repetition_index': 0, 'threads': 1, 'iterations': 75, 'data null%': 1.0,
'extracted_size': 37952800.0, 'mask null%': 5.0, 'num_cols': 100.0, 'select%':
50.0}
FilterFSLInt64FilterNoNulls/4194304/5 13.319 GiB/sec 12.088
GiB/sec -9.246
{'family_index': 2, 'per_family_instance_index': 5, 'run_name':
'FilterFSLInt64FilterNoNulls/4194304/5', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 2365, 'data null%': 0.1, 'mask null%': 0.0,
'select%': 1.0}
FilterInt64FilterWithNulls/4194304/6 2.960 GiB/sec 2.685
GiB/sec -9.291
{'family_index': 1, 'per_family_instance_index': 6, 'run_name':
'FilterInt64FilterWithNulls/4194304/6', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 526, 'data null%': 1.0, 'mask null%': 5.0,
'select%': 99.9}
TakeInt64RandomIndicesWithNulls/4194304/1 1.451G items/sec 1.315G
items/sec -9.434
{'family_index': 9, 'per_family_instance_index': 3, 'run_name':
'TakeInt64RandomIndicesWithNulls/4194304/1', 'repetitions': 1,
'repetition_index': 0, 'threads': 1, 'iterations': 242, 'null_percent': 100.0}
FilterFSLInt64FilterNoNulls/4194304/2 13.907 GiB/sec 12.463
GiB/sec -10.381
{'family_index': 2, 'per_family_instance_index': 2, 'run_name':
'FilterFSLInt64FilterNoNulls/4194304/2', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 2485, 'data null%': 0.0, 'mask null%': 0.0,
'select%': 1.0}
FilterFSLInt64FilterNoNulls/4194304/1 1.274 GiB/sec 1.132
GiB/sec -11.141
{'family_index': 2, 'per_family_instance_index': 1, 'run_name':
'FilterFSLInt64FilterNoNulls/4194304/1', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 230, 'data null%': 0.0, 'mask null%': 0.0,
'select%': 50.0}
FilterStringFilterNoNulls/4194304/10 1.245 GiB/sec 1.100
GiB/sec -11.619 {'family_index':
4, 'per_family_instance_index': 10, 'run_name':
'FilterStringFilterNoNulls/4194304/10', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 227, 'data null%': 10.0, 'mask null%': 0.0,
'select%': 50.0}
FilterStringFilterNoNulls/4194304/9 850.490 MiB/sec 749.768
MiB/sec -11.843
{'family_index': 4, 'per_family_instance_index': 9, 'run_name':
'FilterStringFilterNoNulls/4194304/9', 'repetitions': 1, 'repetition_index': 0,
'threads': 1, 'iterations': 147, 'data null%': 10.0, 'mask null%': 0.0,
'select%': 99.9}
FilterInt64FilterWithNulls/4194304/3 3.212 GiB/sec 2.822
GiB/sec -12.143
{'family_index': 1, 'per_family_instance_index': 3, 'run_name':
'FilterInt64FilterWithNulls/4194304/3', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 570, 'data null%': 0.1, 'mask null%': 5.0,
'select%': 99.9}
FilterStringFilterNoNulls/4194304/6 1.360 GiB/sec 1.192
GiB/sec -12.345
{'family_index': 4, 'per_family_instance_index': 6, 'run_name':
'FilterStringFilterNoNulls/4194304/6', 'repetitions': 1, 'repetition_index': 0,
'threads': 1, 'iterations': 243, 'data null%': 1.0, 'mask null%': 0.0,
'select%': 99.9}
FilterStringFilterWithNulls/4194304/4 1.252 GiB/sec 1.085
GiB/sec -13.320
{'family_index': 5, 'per_family_instance_index': 4, 'run_name':
'FilterStringFilterWithNulls/4194304/4', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 220, 'data null%': 0.1, 'mask null%': 5.0,
'select%': 50.0}
FilterStringFilterNoNulls/4194304/13 277.736 MiB/sec 240.290
MiB/sec -13.483
{'family_index': 4, 'per_family_instance_index': 13, 'run_name':
'FilterStringFilterNoNulls/4194304/13', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 48, 'data null%': 90.0, 'mask null%': 0.0,
'select%': 50.0}
FilterStringFilterWithNulls/4194304/3 976.622 MiB/sec 844.007
MiB/sec -13.579
{'family_index': 5, 'per_family_instance_index': 3, 'run_name':
'FilterStringFilterWithNulls/4194304/3', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 172, 'data null%': 0.1, 'mask null%': 5.0,
'select%': 99.9}
FilterInt64FilterWithNulls/4194304/0 3.268 GiB/sec 2.789
GiB/sec -14.677
{'family_index': 1, 'per_family_instance_index': 0, 'run_name':
'FilterInt64FilterWithNulls/4194304/0', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 567, 'data null%': 0.0, 'mask null%': 5.0,
'select%': 99.9}
FilterStringFilterWithNulls/4194304/1 1.272 GiB/sec 1.084
GiB/sec -14.788
{'family_index': 5, 'per_family_instance_index': 1, 'run_name':
'FilterStringFilterWithNulls/4194304/1', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 221, 'data null%': 0.0, 'mask null%': 5.0,
'select%': 50.0}
FilterStringFilterWithNulls/4194304/6 952.154 MiB/sec 800.961
MiB/sec -15.879
{'family_index': 5, 'per_family_instance_index': 6, 'run_name':
'FilterStringFilterWithNulls/4194304/6', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 165, 'data null%': 1.0, 'mask null%': 5.0,
'select%': 99.9}
FilterStringFilterWithNulls/4194304/10 1.236 GiB/sec 1.038
GiB/sec -16.001 {'family_index':
5, 'per_family_instance_index': 10, 'run_name':
'FilterStringFilterWithNulls/4194304/10', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 223, 'data null%': 10.0, 'mask null%': 5.0,
'select%': 50.0}
FilterStringFilterWithNulls/4194304/7 1.254 GiB/sec 1.052
GiB/sec -16.104
{'family_index': 5, 'per_family_instance_index': 7, 'run_name':
'FilterStringFilterWithNulls/4194304/7', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 223, 'data null%': 1.0, 'mask null%': 5.0,
'select%': 50.0}
FilterStringFilterWithNulls/4194304/0 995.479 MiB/sec 831.870
MiB/sec -16.435
{'family_index': 5, 'per_family_instance_index': 0, 'run_name':
'FilterStringFilterWithNulls/4194304/0', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 174, 'data null%': 0.0, 'mask null%': 5.0,
'select%': 99.9}
FilterStringFilterWithNulls/4194304/9 897.171 MiB/sec 739.897
MiB/sec -17.530 {'family_index':
5, 'per_family_instance_index': 9, 'run_name':
'FilterStringFilterWithNulls/4194304/9', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 157, 'data null%': 10.0, 'mask null%': 5.0,
'select%': 99.9}
FilterStringFilterWithNulls/4194304/13 258.601 MiB/sec 203.408
MiB/sec -21.343 {'family_index':
5, 'per_family_instance_index': 13, 'run_name':
'FilterStringFilterWithNulls/4194304/13', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 45, 'data null%': 90.0, 'mask null%': 5.0,
'select%': 50.0}
FilterStringFilterWithNulls/4194304/12 370.972 MiB/sec 278.844
MiB/sec -24.834 {'family_index':
5, 'per_family_instance_index': 12, 'run_name':
'FilterStringFilterWithNulls/4194304/12', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 65, 'data null%': 90.0, 'mask null%': 5.0,
'select%': 99.9}
FilterStringFilterNoNulls/4194304/12 442.746 MiB/sec 298.930
MiB/sec -32.483
{'family_index': 4, 'per_family_instance_index': 12, 'run_name':
'FilterStringFilterNoNulls/4194304/12', 'repetitions': 1, 'repetition_index':
0, 'threads': 1, 'iterations': 77, 'data null%': 90.0, 'mask null%': 0.0,
'select%': 99.9}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
