Diff
Modified: trunk/Tools/ChangeLog (221776 => 221777)
--- trunk/Tools/ChangeLog 2017-09-08 02:57:23 UTC (rev 221776)
+++ trunk/Tools/ChangeLog 2017-09-08 02:58:08 UTC (rev 221777)
@@ -1,3 +1,58 @@
+2017-09-07 Matthew Stewart <[email protected]>
+
+ Add Live PLT implemented with WebDriver
+ https://bugs.webkit.org/show_bug.cgi?id=176436
+
+ Reviewed by Stephanie Lewis.
+
+ Adds a new PLT that runs on live websites.
+
+ * Scripts/run-webdriver-plt.py: Added.
+ (parse_args):
+ (start):
+ (make_suites):
+ (main):
+ * Scripts/webkitpy/webdriver_plt/__init__.py: Added.
+ * Scripts/webkitpy/webdriver_plt/liveplt.py: Added.
+ (PageLoadTest):
+ (PageLoadTest.__init__):
+ (PageLoadTest.start):
+ (PageLoadTest.run_suite):
+ (PageLoadTest._get_driver_for_browser):
+ (PageLoadTest._setup_browser_window):
+ (PageLoadTest.run_one_test):
+ (PageLoadTest.get_results):
+ * Scripts/webkitpy/webdriver_plt/pltresults.py: Added.
+ (PLTResults):
+ (PLTResults.__init__):
+ (PLTResults.__add__):
+ (PLTResults.add_timing_result):
+ (PLTResults.mean):
+ (PLTResults.geometric_mean):
+ (PLTResults.mean_coef_variance):
+ (PLTResults.print_results):
+ (PLTResults.print_url_results):
+ (PLTResults._format_time):
+ * Scripts/webkitpy/webdriver_plt/suites/__init__.py: Added.
+ * Scripts/webkitpy/webdriver_plt/suites/arabic.suite: Added.
+ * Scripts/webkitpy/webdriver_plt/suites/cjk.suite: Added.
+ * Scripts/webkitpy/webdriver_plt/suites/news.suite: Added.
+ * Scripts/webkitpy/webdriver_plt/suites/search.suite: Added.
+ * Scripts/webkitpy/webdriver_plt/suites/shopping.suite: Added.
+ * Scripts/webkitpy/webdriver_plt/suites/social.suite: Added.
+ * Scripts/webkitpy/webdriver_plt/suites/suite.py: Added.
+ (Suite):
+ (Suite.__init__):
+ (Suite.get_available_suites):
+ * Scripts/webkitpy/webdriver_plt/urlresults.py: Added.
+ (URLResults):
+ (URLResults.__init__):
+ (URLResults.__add__):
+ (URLResults.mean):
+ (URLResults.coef_variance):
+ (URLResults.print_results):
+ (URLResults._format_time):
+
2017-09-07 Myles C. Maxfield <[email protected]>
Add "if" statements to WSL
Added: trunk/Tools/Scripts/run-webdriver-plt.py (0 => 221777)
--- trunk/Tools/Scripts/run-webdriver-plt.py (rev 0)
+++ trunk/Tools/Scripts/run-webdriver-plt.py 2017-09-08 02:58:08 UTC (rev 221777)
@@ -0,0 +1,52 @@
+import argparse
+from webkitpy.webdriver_plt.suites.suite import Suite
+from webkitpy.webdriver_plt.liveplt import PageLoadTest
+
+available_browsers = [
+ "safari", "chrome", "firefox", "stp",
+]
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Automate running PLT with live websites')
+ parser.add_argument('-i', '--iterations', dest='iterations', type=int, default=3, help='Test the suite i times in the same browser session')
+ parser.add_argument('-n', '--instances', dest='instances', type=int, default=3, help='Restart the browser n times for each suite')
+ parser.add_argument('-w', '--wait', dest='wait', type=float, default=3.0, help='Wait time between pages')
+ parser.add_argument('-b', '--browser', dest='browser', default='safari', choices=available_browsers)
+ parser.add_argument('-s', '--suites', dest='suites', nargs='+', help='List one or more suites to run. If unspecified, defaults to running all suites')
+ parser.add_argument('--width', dest='width', help='Set the inner window width')
+ parser.add_argument('--height', dest='height', help='Set the inner window height')
+
+ args = parser.parse_args()
+
+ return args
+
+
+def start(args):
+ suites = make_suites(args.suites)
+ size = (args.width, args.height)
+ plt = PageLoadTest(args.iterations, args.instances, args.wait, args.browser, suites, size)
+ plt.start()
+
+
+def make_suites(suiteslist):
+ available_suites = Suite.get_available_suites()
+ suites = list()
+ if suiteslist:
+ for suitename in suiteslist:
+ if suitename.lower() not in available_suites:
+ print("Suite \"{}\" not found.".format(suitename))
+ quit()
+ suites.append(Suite(suitename.lower()))
+ else:
+ for suitename in available_suites:
+ suites.append(Suite(suitename))
+ return suites
+
+
+def main():
+ return start(parse_args())
+
+
+if __name__ == '__main__':
+ main()
Property changes on: trunk/Tools/Scripts/run-webdriver-plt.py
___________________________________________________________________
Added: svn:executable
+*
\ No newline at end of property
Added: trunk/Tools/Scripts/webkitpy/webdriver_plt/__init__.py (0 => 221777)
--- trunk/Tools/Scripts/webkitpy/webdriver_plt/__init__.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/webdriver_plt/__init__.py 2017-09-08 02:58:08 UTC (rev 221777)
@@ -0,0 +1 @@
+pass
Added: trunk/Tools/Scripts/webkitpy/webdriver_plt/liveplt.py (0 => 221777)
--- trunk/Tools/Scripts/webkitpy/webdriver_plt/liveplt.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/webdriver_plt/liveplt.py 2017-09-08 02:58:08 UTC (rev 221777)
@@ -0,0 +1,172 @@
+import webkitpy.thirdparty.autoinstalled.selenium
+from pltresults import PLTResults
+from selenium import webdriver
+from time import sleep
+from webkitpy.benchmark_runner.utils import get_driver_binary_path
+from webkitpy.common.timeout_context import Timeout
+
+# FIXME: we want to avoid hardcoding the offsets
+# we use this to make sure the window.innerHeight of each browser is the same
+browser_height_offsets = {
+ 'safari': 76,
+ 'chrome': 0,
+ 'firefox': 37,
+ 'stp': 76,
+}
+
+
+class PageLoadTest(object):
+
+ def __init__(self, iterations, instances, wait, browser, suites, size):
+ print("Creating PLT with parameters:")
+ print("iterations: {}".format(iterations))
+ print("instances: {}".format(instances))
+ print("wait: {}s".format(wait))
+ print("browser: {}".format(browser))
+ print("suites: {}".format([s.name for s in suites]))
+ width, height = size
+ if width:
+ print("width: {}".format(width))
+ if height:
+ print("height: {}".format(height))
+ print("")
+ self.iterations = iterations
+ self.instances = instances
+ self.wait = wait
+ self.browser = browser
+ self.suites = suites
+ self.size = size
+
+ def start(self):
+ total_results = PLTResults()
+ suiteid = 0
+ for suite in self.suites:
+ suite.attempts -= 1
+ suiteid += 1
+ print("------------------------------")
+ print("Running suite {id} of {num}: {name}".format(id=suiteid, num=len(self.suites), name=suite.name))
+ print("------------------------------")
+ print("")
+ total_results += self.run_suite(suite)
+ total_results.print_results()
+
+ def run_suite(self, suite):
+ cold_run_results = PLTResults()
+ suite_results = PLTResults()
+
+ for instance in range(self.instances):
+ driver = self._get_driver_for_browser(self.browser)
+ self._setup_browser_window(driver)
+ sleep(5)
+ for iteration in range(self.iterations + 1):
+ print("------------------------------")
+ if iteration == 0:
+ print("Running iternation %s (cold run)" % iteration)
+ else:
+ print("Running iternation %s" % iteration)
+ run_results = self.run_one_test(suite, driver)
+ if not run_results:
+ if suite.attempts <= 0:
+ print("Failed to finish suite {name} after {x} attempts. Results are incomplete.".format(name=suite.name, x=suite.max_attempts))
+ print("Exiting...")
+ quit()
+ else:
+ print("A page failed to load. Re-queuing suite.")
+ self.suites.append(suite)
+ driver.quit()
+ return PLTResults()
+ run_results.print_url_results("INDIVIDUAL")
+ if iteration == 0:
+ cold_run_results += run_results
+ else:
+ suite_results += run_results
+ driver.quit()
+ cold_run_results.print_results(suite.name, True)
+ suite_results.print_results(suite.name)
+ return suite_results
+
+ def _get_driver_for_browser(self, browser):
+ driver_executable = get_driver_binary_path(browser)
+ if browser == 'safari':
+ return webdriver.Safari()
+ if browser == 'chrome':
+ from selenium.webdriver.chrome.options import Options
+ options = Options()
+ options.add_argument("--disable-web-security")
+ options.add_argument("--disable-extensions")
+ options.add_argument("--start-maximized")
+ return webdriver.Chrome(chrome_options=options, executable_path=driver_executable)
+ if browser == 'firefox':
+ return webdriver.Firefox(executable_path=driver_executable)
+ if browser == 'stp':
+ return webdriver.Safari(executable_path='/Applications/Safari Technology Preview.app/Contents/MacOS/safaridriver')
+
+ def _setup_browser_window(self, driver):
+ driver.maximize_window()
+ sleep(1)
+ current_size = driver.get_window_size()
+
+ new_width, new_height = self.size
+ if not new_width:
+ new_width = driver.execute_script('return screen.width;')
+ print("setting width to {}".format(new_width))
+ if not new_height:
+ new_height = current_size['height'] - browser_height_offsets[self.browser]
+ print("setting height to {}".format(new_height))
+
+ try:
+ driver.set_window_size(width=new_width, height=new_height)
+ driver.set_window_position(x=0, y=0)
+ except:
+ pass
+
+ def run_one_test(self, suite, driver):
+ tabs = False
+ currentTabIdx = 0
+ maxTabs = 5
+ maxRetrys = 5
+ timeout = 30
+
+ local_results = PLTResults()
+
+ for url in suite.urls:
+ if tabs:
+ if currentTabIdx < maxTabs - 1:
+ raise NotImplementedError("Opening new tabs is not supported yet.")
+ driver.switch_to_window(driver.window_handles[currentTabIdx % maxTabs])
+ currentTabIdx += 1
+ tempWait = self.wait
+ for attempt in range(maxRetrys):
+ driver.set_page_load_timeout(timeout)
+ try:
+ driver.get(url)
+ except:
+ print("{url} timed out after {time} seconds.".format(url="" time=timeout))
+ return None
+
+ sleep(tempWait)
+ if self.get_results(driver, local_results, url):
+ break
+
+ print("Could not get results for {url}. Retrying...".format(url=""
+ tempWait += 0.5
+ if attempt == maxRetrys - 1:
+ return None
+ return local_results
+
+ def get_results(self, driver, results, url):
+ nt_results = driver.execute_script('return performance.getEntriesByType("navigation")[0]') # navigation timing 2
+ if not nt_results:
+ nt_results = driver.execute_script('return performance.timing') # navigation timing 1
+ start = nt_results['navigationStart']
+ end = nt_results['loadEventEnd']
+ if start == 0 or end == 0:
+ return False
+ else:
+ start = nt_results['startTime']
+ end = nt_results['loadEventEnd']
+ if end == 0:
+ return False
+
+ results.add_timing_result(end - start, url)
+ return True
Added: trunk/Tools/Scripts/webkitpy/webdriver_plt/pltresults.py (0 => 221777)
--- trunk/Tools/Scripts/webkitpy/webdriver_plt/pltresults.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/webdriver_plt/pltresults.py 2017-09-08 02:58:08 UTC (rev 221777)
@@ -0,0 +1,100 @@
+from collections import OrderedDict
+from urlresults import URLResults
+from math import log, exp
+
+
+class PLTResults(object):
+
+ def __init__(self, geo_sum=1, totalsum=0, num_urls=0, urls=None):
+ self.geo_sum = geo_sum
+ self.sum = totalsum
+ self.count = num_urls
+ if urls is None:
+ urls = OrderedDict()
+ self.urls = urls
+
+ def __add__(self, other):
+ new_geo_sum = self.geo_sum + other.geo_sum
+ new_sum = self.sum + other.sum
+ new_count = self.count + other.count
+ new_urls = OrderedDict(self.urls)
+ for (url, results) in other.urls.items():
+ if url not in self.urls:
+ new_urls[url] = URLResults()
+ new_urls[url] += results
+ return PLTResults(new_geo_sum, new_sum, new_count, new_urls)
+
+ def add_timing_result(self, time, url):
+ self.geo_sum += log(time)
+ self.sum += time
+ self.count += 1
+ if url in self.urls:
+ self.urls[url] += URLResults(time, 1, time ** 2)
+ else:
+ self.urls[url] = URLResults(time, 1, time ** 2)
+
+ @property
+ def mean(self):
+ if self.count:
+ return self.sum / float(self.count)
+ else:
+ return 0
+
+ @property
+ def geometric_mean(self):
+ if self.count:
+ return exp(self.geo_sum / float(self.count))
+ else:
+ return 0
+
+ @property
+ def mean_coef_variance(self):
+ total = 0.0
+ for _, results in self.urls.items():
+ total += results.coef_variance
+ if self.urls and len(self.urls):
+ return total / float(len(self.urls))
+ else:
+ return 0.0
+
+ def print_results(self, suitename=None, cold_run=False):
+ print("------------------------------")
+ if not suitename:
+ print("")
+ print("------------------------------")
+ print("OVERALL PLT RESULTS SUMMARY:")
+ elif cold_run:
+ print("Cold Run Results for {name}:".format(name=suitename))
+ else:
+ print("PLT Results for {name}:".format(name=suitename))
+ if suitename:
+ self.print_url_results("MEAN")
+ else:
+ self.print_url_results("TOTAL MEAN")
+ print("------------------------------")
+ print("TOTAL TIME: {time}".format(time=self._format_time(self.sum)))
+ print("TOTAL URLS: {urls}".format(urls=self.count))
+ if self.urls and len(self.urls) and self.urls.values()[0].count > 1:
+ print("AVERAGE COEFFICIENT OF VARIANCE: {0:.2f}%".format(self.mean_coef_variance))
+ print("AVERAGE TIME PER PAGE: {avg}".format(avg=self._format_time(self.mean)))
+ print("GEOMETRIC MEAN: {avg}".format(avg=self._format_time(self.geometric_mean)))
+ print("------------------------------")
+ print("")
+
+ def print_url_results(self, urlResultsType):
+ print("------------------------------")
+ print("{type} URL LOAD TIMES:".format(type=urlResultsType))
+ if not len(self.urls):
+ print("")
+ return
+ if self.urls.values()[0].count > 1:
+ results_format_str = "{time}{var}{url}"
+ else:
+ results_format_str = "{time}{url}"
+ print(results_format_str.format(time="time (ms)".ljust(12), var="CoV (%)".ljust(10), url=""
+ for (url, results) in self.urls.items():
+ results.print_results(url, results_format_str)
+ print("")
+
+ def _format_time(self, time):
+ return "{} ms".format("{0:.2f}".format(float(time)).rstrip('0').rstrip('.'))
Added: trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/__init__.py (0 => 221777)
--- trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/__init__.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/__init__.py 2017-09-08 02:58:08 UTC (rev 221777)
@@ -0,0 +1 @@
+pass
Added: trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/arabic.suite (0 => 221777)
--- trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/arabic.suite (rev 0)
+++ trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/arabic.suite 2017-09-08 02:58:08 UTC (rev 221777)
@@ -0,0 +1,14 @@
+{
+ "urls": [
+ "http://www.kooora.com",
+ "http://www.kooora.com/?n=0&o=s0",
+ "https://www.bayt.com/ar",
+ "https://www.bayt.com/ar/employers",
+ "http://www.yallakora.com",
+ "http://www.yallakora.com/ar/tours",
+ "http://www.un.org/ar/index.html",
+ "http://www.un.org/ar/sections/resources/delegates",
+ "https://www.emirates.com/ae/arabic",
+ "https://www.emirates.com/ae/arabic/destinations/flights-to-united-states.aspx"
+ ]
+}
Added: trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/cjk.suite (0 => 221777)
--- trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/cjk.suite (rev 0)
+++ trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/cjk.suite 2017-09-08 02:58:08 UTC (rev 221777)
@@ -0,0 +1,16 @@
+{
+ "urls": [
+ "https://www.sogou.com",
+ "https://www.sogou.com/web?query=webkit",
+ "https://cn.nytimes.com",
+ "https://cn.nytimes.com/usa",
+ "https://www.yahoo.co.jp",
+ "https://news.yahoo.co.jp/pickup/6252672",
+ "https://www.rakuten.co.jp",
+ "https://event.rakuten.co.jp",
+ "http://kakaku.com",
+ "http://kakaku.com/pc/lcd-monitor",
+ "https://www.naver.com",
+ "https://section.blog.naver.com"
+ ]
+}
Added: trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/news.suite (0 => 221777)
--- trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/news.suite (rev 0)
+++ trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/news.suite 2017-09-08 02:58:08 UTC (rev 221777)
@@ -0,0 +1,14 @@
+{
+ "urls": [
+ "https://www.nytimes.com",
+ "https://www.nytimes.com/2017/06/27/technology/education-partovi-computer-science-coding-apple-microsoft.html",
+ "https://www.theguardian.com/us/technology",
+ "https://www.theguardian.com/technology/2017/jun/29/iphone-at-10-how-it-changed-everything",
+ "http://www.espn.com",
+ "http://www.espn.com/tennis/story/_/id/20466488/rafael-nadal-does-not-want-face-roger-federer-us-open",
+ "https://news.google.com/news",
+ "https://news.google.com/news/headlines/section/topic/WORLD",
+ "https://www.yahoo.com",
+ "https://www.yahoo.com/tech/apos-pok-mon-apos-celebrates-155523814.html"
+ ]
+}
Added: trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/search.suite (0 => 221777)
--- trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/search.suite (rev 0)
+++ trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/search.suite 2017-09-08 02:58:08 UTC (rev 221777)
@@ -0,0 +1,14 @@
+{
+ "urls": [
+ "https://www.google.com",
+ "https://www.google.com/search?q=webkit",
+ "https://search.yahoo.com",
+ "https://search.yahoo.com/search?p=webkit",
+ "https://www.yandex.com",
+ "https://www.yandex.com/search/?text=webkit",
+ "http://www.bing.com",
+ "http://www.bing.com/search?q=webkit",
+ "https://duckduckgo.com",
+ "https://duckduckgo.com/?q=webkit"
+ ]
+}
Added: trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/shopping.suite (0 => 221777)
--- trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/shopping.suite (rev 0)
+++ trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/shopping.suite 2017-09-08 02:58:08 UTC (rev 221777)
@@ -0,0 +1,14 @@
+{
+ "urls": [
+ "https://www.amazon.com",
+ "https://www.amazon.com/Even-Faster-Web-Sites-Performance/dp/0596522304",
+ "https://www.ebay.com",
+ "https://www.ebay.com/deals",
+ "https://www.craigslist.org",
+ "https://www.craigslist.org/search/sss",
+ "https://www.walmart.com",
+ "https://www.walmart.com/search/?query=food",
+ "https://www.etsy.com",
+ "https://www.etsy.com/c/electronics-and-accessories"
+ ]
+}
Added: trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/social.suite (0 => 221777)
--- trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/social.suite (rev 0)
+++ trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/social.suite 2017-09-08 02:58:08 UTC (rev 221777)
@@ -0,0 +1,16 @@
+{
+ "urls": [
+ "https://www.facebook.com",
+ "https://www.facebook.com/facebook",
+ "https://www.tumblr.com/search/apple",
+ "http://rediscoveringearth.tumblr.com",
+ "https://twitter.com/search?q=webkit",
+ "https://twitter.com/webkit",
+ "https://www.instagram.com/explore/tags/sky",
+ "https://www.instagram.com/apple",
+ "https://www.reddit.com",
+ "https://www.reddit.com/r/programming",
+ "https://www.youtube.com",
+ "https://www.youtube.com/watch?v=Q6dsRpVyyWs"
+ ]
+}
Added: trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/suite.py (0 => 221777)
--- trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/suite.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/webdriver_plt/suites/suite.py 2017-09-08 02:58:08 UTC (rev 221777)
@@ -0,0 +1,22 @@
+import json
+import os
+
+
+class Suite(object):
+ def __init__(self, suitename):
+ filename = "{0}.suite".format(suitename)
+ filepath = os.path.join(os.path.dirname(__file__), filename)
+ with open(filepath) as suitefile:
+ _suite = json.load(suitefile)
+ self.urls = _suite['urls']
+ self.name = suitename
+ self.attempts = 2
+ self.max_attempts = 2
+
+ @classmethod
+ def get_available_suites(cls):
+ suitesdir = os.path.dirname(__file__)
+ suiteslist = [os.path.splitext(f)[0].lower() for f in os.listdir(suitesdir) if f.endswith('.suite')]
+ if not suiteslist:
+ raise Exception('Cant find any .suite files in directory %s' % suitesdir)
+ return suiteslist
Added: trunk/Tools/Scripts/webkitpy/webdriver_plt/urlresults.py (0 => 221777)
--- trunk/Tools/Scripts/webkitpy/webdriver_plt/urlresults.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/webdriver_plt/urlresults.py 2017-09-08 02:58:08 UTC (rev 221777)
@@ -0,0 +1,37 @@
+from math import sqrt
+
+
+class URLResults(object):
+
+ def __init__(self, totaltime=0, num_urls=0, std_dev=0.0):
+ self.time = totaltime
+ self.count = num_urls
+ self.std_dev_counter = std_dev
+
+ def __add__(self, other):
+ new_time = self.time + other.time
+ new_count = self.count + other.count
+ new_std_dev_counter = self.std_dev_counter + other.std_dev_counter
+ return URLResults(new_time, new_count, new_std_dev_counter)
+
+ @property
+ def mean(self):
+ if self.count:
+ return self.time / float(self.count)
+ else:
+ return 0
+
+ @property
+ def coef_variance(self):
+ n = self.count
+ if n <= 1:
+ return 0
+ std_dev = sqrt((self.std_dev_counter / float(n - 1)) - (n / float(n - 1)) * (self.mean ** 2))
+ return (std_dev * 100.0) / self.mean
+
+ def print_results(self, url, results_format_str):
+ str_time = self._format_time(self.mean)
+ print(results_format_str.format(time=str_time.ljust(12), var="{0:.2f}%".format(self.coef_variance).ljust(10), url=""
+
+ def _format_time(self, time):
+ return "{} ms".format(int(time))
