This is an automated email from the ASF dual-hosted git repository.
yibocai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new a542e52 ARROW-12145: [Developer][Archery] Flaky:
test_static_runner_from_json
a542e52 is described below
commit a542e524e1f65e22ecda5bfbf8147bdbe85dbc87
Author: Diana Clarke <[email protected]>
AuthorDate: Wed Mar 31 01:37:47 2021 +0000
ARROW-12145: [Developer][Archery] Flaky: test_static_runner_from_json
This test assumes:
```
artificial_reg, normal = RunnerComparator(contender, baseline).comparisons
```
When the return order could be:
```
normal, artificial_reg = RunnerComparator(contender, baseline).comparisons
```
The return order of `comparisons` isn't deterministic.
See: https://issues.apache.org/jira/browse/ARROW-12145
Closes #9843 from dianaclarke/ARROW-12145
Authored-by: Diana Clarke <[email protected]>
Signed-off-by: Yibo Cai <[email protected]>
---
dev/archery/archery/tests/test_benchmarks.py | 36 +++++++++++++++++++---------
1 file changed, 25 insertions(+), 11 deletions(-)
diff --git a/dev/archery/archery/tests/test_benchmarks.py
b/dev/archery/archery/tests/test_benchmarks.py
index 52be813..dffe698 100644
--- a/dev/archery/archery/tests/test_benchmarks.py
+++ b/dev/archery/archery/tests/test_benchmarks.py
@@ -52,8 +52,7 @@ def test_benchmark_comparator():
).regression
-def test_static_runner_from_json():
- # full output of `archery benchmark run`
+def test_static_runner_from_json_not_a_regression():
archery_result = {
"suites": [
{
@@ -71,18 +70,35 @@ def test_static_runner_from_json():
9095.800104330105
]
},
+ ]
+ }
+ ]
+ }
+
+ contender = StaticBenchmarkRunner.from_json(json.dumps(archery_result))
+ baseline = StaticBenchmarkRunner.from_json(json.dumps(archery_result))
+ [comparison] = RunnerComparator(contender, baseline).comparisons
+ assert not comparison.regression
+
+
+def test_static_runner_from_json_regression():
+ archery_result = {
+ "suites": [
+ {
+ "name": "arrow-value-parsing-benchmark",
+ "benchmarks": [
{
- "name": "FloatParsing<FloatType>",
+ "name": "FloatParsing<DoubleType>",
"unit": "items_per_second",
"less_is_better": False,
"values": [
- 105982641.9337845
+ 109941112.87296811
],
"time_unit": "ns",
"times": [
- 9435.567922160235
+ 9095.800104330105
]
- }
+ },
]
}
]
@@ -90,14 +106,12 @@ def test_static_runner_from_json():
contender = StaticBenchmarkRunner.from_json(json.dumps(archery_result))
- # introduce artificial regression:
+ # introduce artificial regression
archery_result['suites'][0]['benchmarks'][0]['values'][0] *= 2
baseline = StaticBenchmarkRunner.from_json(json.dumps(archery_result))
- artificial_reg, normal = RunnerComparator(contender, baseline).comparisons
-
- assert artificial_reg.regression
- assert not normal.regression
+ [comparison] = RunnerComparator(contender, baseline).comparisons
+ assert comparison.regression
def test_benchmark_median():
