Modified: trunk/Tools/ChangeLog (279558 => 279559)
--- trunk/Tools/ChangeLog 2021-07-04 19:18:38 UTC (rev 279558)
+++ trunk/Tools/ChangeLog 2021-07-04 21:06:12 UTC (rev 279559)
@@ -1,3 +1,23 @@
+2021-07-04 Ben Nham <[email protected]>
+
+ Add support for parsing Competitive PLT results to compare-results
+ https://bugs.webkit.org/show_bug.cgi?id=227470
+
+ Reviewed by Per Arne Vollan.
+
+ Modify the script to understand the competitive PLT JSON results produced by the perf bots.
+
+ * Scripts/compare-results:
+ (plt5Breakdown):
+ (competitivePLTBreakdown):
+ (PLT5Results):
+ (detectCompetitivePLT):
+ (CompetitivePLTResults):
+ (CompetitivePLTResults.calculate_time_for_run):
+ (detectBenchmark):
+ (biggerIsBetter):
+ (main):
+
2021-07-02 Aakash Jain <[email protected]>
Delete unused BuildSlaveSupport symlink
Modified: trunk/Tools/Scripts/compare-results (279558 => 279559)
--- trunk/Tools/Scripts/compare-results 2021-07-04 19:18:38 UTC (rev 279558)
+++ trunk/Tools/Scripts/compare-results 2021-07-04 21:06:12 UTC (rev 279559)
@@ -26,6 +26,7 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import collections
import sys
import argparse
import json
@@ -55,6 +56,7 @@
Speedometer2 = "Speedometer2"
JetStream2 = "JetStream2"
PLT5 = "PLT5"
+CompetitivePLT = "CompetitivePLT"
PLUM3 = "PLUM3"
MotionMark = "MotionMark"
MotionMark1_1 = "MotionMark-1.1"
@@ -122,6 +124,30 @@
return result
+def competitivePLTBreakdown(jsonObject):
+ result = collections.defaultdict(list)
+ result[unitMarker] = "sec"
+
+ safari_results = jsonObject.get('Safari', {})
+
+ cold_results = safari_results.get('cold', {})
+ warm_results = safari_results.get('warm', {})
+
+ cold_link_results = cold_results.get('add-and-click-link', {})
+ warm_link_results = warm_results.get('add-and-click-link', {})
+
+ for site_to_times in cold_link_results.values():
+ for site, times in site_to_times.items():
+ result["cold--fmp--" + site].append(times['first_meaningful_paint'])
+ result["cold--load-end--" + site].append(times['load_end'])
+
+ for site_to_times in warm_link_results.values():
+ for site, times in site_to_times.items():
+ result["warm--fmp--" + site].append(times['first_meaningful_paint'])
+ result["warm--load-end--" + site].append(times['load_end'])
+
+ return result
+
def plum3Breakdown(jsonObject):
breakdown = BenchmarkResults(jsonObject)
result = {}
@@ -319,6 +345,29 @@
results.append(obj["Geometric"])
return results
+def detectCompetitivePLT(payload):
+ return 'add-and-click-link' in payload.get('Safari', {}).get('cold', {})
+
+def CompetitivePLTResults(payload):
+ def calculate_time_for_run(run):
+ # We geomean all FMP and load_end times together to produce a result for the run.
+ fmp_vals = [obj['first_meaningful_paint'] for obj in run.values()]
+ load_end_vals = [obj['load_end'] for obj in run.values()]
+ return stats.gmean(fmp_vals + load_end_vals)
+
+ safari_results = payload.get('Safari', {})
+
+ cold_results = safari_results.get('cold', {})
+ warm_results = safari_results.get('warm', {})
+
+ cold_link_results = cold_results.get('add-and-click-link', {})
+ warm_link_results = warm_results.get('add-and-click-link', {})
+
+ cold_times = [calculate_time_for_run(run) for run in cold_link_results.values()]
+ warm_times = [calculate_time_for_run(run) for run in warm_link_results.values()]
+
+ return [stats.gmean((cold_time, warm_time)) for cold_time, warm_time in zip(cold_times, warm_times)]
+
def detectPLUM3(payload):
return "PLUM3-PhysFootprint" in payload
@@ -362,6 +411,8 @@
return Speedometer2
if detectPLT5(payload):
return PLT5
+ if detectCompetitivePLT(payload):
+ return CompetitivePLT
if detectPLUM3(payload):
return PLUM3
if detectMotionMark(payload):
@@ -383,6 +434,8 @@
return True
if benchmarkType == PLT5:
return False
+ if benchmarkType == CompetitivePLT:
+ return False
if benchmarkType == PLUM3:
return False
@@ -499,6 +552,14 @@
if args.csv:
writeCSV(plt5Breakdown(a), plt5Breakdown(b), args.csv)
+ elif typeA == CompetitivePLT:
+ if args.breakdown:
+ dumpBreakdowns(competitivePLTBreakdown(a), competitivePLTBreakdown(b))
+
+ ttest(typeA, CompetitivePLTResults(a), CompetitivePLTResults(b))
+
+ if args.csv:
+ writeCSV(competitivePLTBreakdown(a), competitivePLTBreakdown(b), args.csv)
elif typeA == PLUM3:
if args.breakdown:
dumpBreakdowns(plum3Breakdown(a), plum3Breakdown(b))
