Diff
Modified: trunk/ChangeLog (107273 => 107274)
--- trunk/ChangeLog 2012-02-09 20:18:48 UTC (rev 107273)
+++ trunk/ChangeLog 2012-02-09 20:20:41 UTC (rev 107274)
@@ -1,3 +1,41 @@
+2012-02-09 Ryosuke Niwa <[email protected]>
+
+ Perf-o-matic should update memcache in taskqueue
+ https://bugs.webkit.org/show_bug.cgi?id=78209
+
+ Reviewed by Adam Barth.
+
+ Update dashboard, manifest, and runs memcaches in taskqueue.
+ Also centralized the management of caches in controller.py.
+
+ * Websites/webkit-perf.appspot.com/app.yaml:
+ * Websites/webkit-perf.appspot.com/controller.py: Added.
+ (cache_manifest):
+ (schedule_manifest_update):
+ (CachedManifestHandler):
+ (CachedManifestHandler.get):
+ (cache_dashboard):
+ (schedule_dashboard_update):
+ (CachedDashboardHandler):
+ (CachedDashboardHandler.get):
+ (cache_runs):
+ (schedule_runs_update):
+ (CachedRunsHandler):
+ (CachedRunsHandler.get):
+ * Websites/webkit-perf.appspot.com/create_handler.py:
+ (CreateHandler.post):
+ * Websites/webkit-perf.appspot.com/dashboard_handler.py:
+ (DashboardHandler.post):
+ * Websites/webkit-perf.appspot.com/main.py:
+ * Websites/webkit-perf.appspot.com/manifest_handler.py:
+ (ManifestHandler.post):
+ * Websites/webkit-perf.appspot.com/merge_tests_handler.py:
+ (MergeTestsHandler.post):
+ * Websites/webkit-perf.appspot.com/report_handler.py:
+ (ReportHandler.post):
+ * Websites/webkit-perf.appspot.com/runs_handler.py:
+ (RunsHandler.post):
+
2012-02-09 Carlos Garcia Campos <[email protected]>
[GTK] Add WebKitWebView::mouse-target-changed signal to WebKit2 GTK+ API
Modified: trunk/Websites/webkit-perf.appspot.com/app.yaml (107273 => 107274)
--- trunk/Websites/webkit-perf.appspot.com/app.yaml 2012-02-09 20:18:48 UTC (rev 107273)
+++ trunk/Websites/webkit-perf.appspot.com/app.yaml 2012-02-09 20:20:41 UTC (rev 107274)
@@ -1,5 +1,5 @@
application: webkit-perf
-version: 11
+version: 12
runtime: python27
api_version: 1
threadsafe: false
Added: trunk/Websites/webkit-perf.appspot.com/controller.py (0 => 107274)
--- trunk/Websites/webkit-perf.appspot.com/controller.py (rev 0)
+++ trunk/Websites/webkit-perf.appspot.com/controller.py 2012-02-09 20:20:41 UTC (rev 107274)
@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+# Copyright (C) 2012 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import webapp2
+from google.appengine.api import memcache
+from google.appengine.api import taskqueue
+
+from models import Test
+
+
+def cache_manifest(cache):
+ memcache.set('manifest', cache)
+
+
+def schedule_manifest_update():
+ taskqueue.add(url='')
+
+
+class CachedManifestHandler(webapp2.RequestHandler):
+ def get(self):
+ self.response.headers['Content-Type'] = 'application/json'
+ manifest = memcache.get('manifest')
+ if manifest:
+ self.response.out.write(manifest)
+ else:
+ schedule_manifest_update()
+
+
+def cache_dashboard(cache):
+ memcache.set('dashboard', cache)
+
+
+def schedule_dashboard_update():
+ taskqueue.add(url='')
+
+
+class CachedDashboardHandler(webapp2.RequestHandler):
+ def get(self):
+ self.response.headers['Content-Type'] = 'application/json'
+ dashboard = memcache.get('dashboard')
+ if dashboard:
+ self.response.out.write(dashboard)
+ else:
+ schedule_dashboard_update()
+
+
+def cache_runs(test_id, branch_id, platform_id, cache):
+ memcache.set(Test.cache_key(test_id, branch_id, platform_id), cache)
+
+
+def schedule_runs_update(test_id, branch_id, platform_id):
+ taskqueue.add(url='', params={'id': test_id, 'branchid': branch_id, 'platformid': platform_id})
+
+
+class CachedRunsHandler(webapp2.RequestHandler):
+ def get(self):
+ self.response.headers['Content-Type'] = 'application/json'
+
+ try:
+ test_id = int(self.request.get('id', 0))
+ branch_id = int(self.request.get('branchid', 0))
+ platform_id = int(self.request.get('platformid', 0))
+ except TypeError:
+ # FIXME: Output an error here
+ test_id = 0
+ branch_id = 0
+ platform_id = 0
+
+ runs = memcache.get(Test.cache_key(test_id, branch_id, platform_id))
+ if runs:
+ self.response.out.write(runs)
+ else:
+ schedule_runs_update(test_id, branch_id, platform_id)
Modified: trunk/Websites/webkit-perf.appspot.com/create_handler.py (107273 => 107274)
--- trunk/Websites/webkit-perf.appspot.com/create_handler.py 2012-02-09 20:18:48 UTC (rev 107273)
+++ trunk/Websites/webkit-perf.appspot.com/create_handler.py 2012-02-09 20:20:41 UTC (rev 107274)
@@ -28,11 +28,11 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import webapp2
-from google.appengine.api import memcache
from google.appengine.ext import db
import json
+from controller import schedule_dashboard_update
from models import Builder
from models import Branch
from models import NumericIdHolder
@@ -42,7 +42,7 @@
class CreateHandler(webapp2.RequestHandler):
def post(self, model):
- self.response.headers['Content-Type'] = 'text/plain; charset=utf-8';
+ self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
try:
payload = json.loads(self.request.body)
@@ -63,7 +63,7 @@
error = "Unknown model type: %s\n" % model
# No need to clear manifest or runs since they only contain ones with test results
- memcache.delete('dashboard')
+ schedule_dashboard_update()
self.response.out.write(error + '\n' if error else 'OK')
def _create_builder(self, name, password):
Modified: trunk/Websites/webkit-perf.appspot.com/dashboard_handler.py (107273 => 107274)
--- trunk/Websites/webkit-perf.appspot.com/dashboard_handler.py 2012-02-09 20:18:48 UTC (rev 107273)
+++ trunk/Websites/webkit-perf.appspot.com/dashboard_handler.py 2012-02-09 20:20:41 UTC (rev 107274)
@@ -28,10 +28,9 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import webapp2
-from google.appengine.api import memcache
-
import json
+from controller import cache_dashboard
from models import Builder
from models import Branch
from models import Platform
@@ -39,15 +38,13 @@
class DashboardHandler(webapp2.RequestHandler):
- def get(self):
- self.response.headers['Content-Type'] = 'application/json; charset=utf-8';
- cache = memcache.get('dashboard')
- if cache:
- self.response.out.write(cache)
+ def post(self):
+ self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
+ webkit_trunk = Branch.get_by_key_name('webkit-trunk')
+ if not webkit_trunk:
+ self.response.out.write("BAD: no webkit-trunk")
return
- webkit_trunk = Branch.get_by_key_name('webkit-trunk')
-
# FIXME: Determine popular branches, platforms, and tests
dashboard = {
'defaultBranch': 'WebKit trunk',
@@ -62,6 +59,5 @@
for test in Test.all():
dashboard['testToId'][test.name] = test.id
- result = json.dumps(dashboard)
- self.response.out.write(result)
- memcache.add('dashboard', result)
+ cache_dashboard(json.dumps(dashboard))
+ self.response.out.write('OK')
Modified: trunk/Websites/webkit-perf.appspot.com/main.py (107273 => 107274)
--- trunk/Websites/webkit-perf.appspot.com/main.py 2012-02-09 20:18:48 UTC (rev 107273)
+++ trunk/Websites/webkit-perf.appspot.com/main.py 2012-02-09 20:20:41 UTC (rev 107274)
@@ -20,6 +20,9 @@
import json
+from controller import CachedDashboardHandler
+from controller import CachedManifestHandler
+from controller import CachedRunsHandler
from create_handler import CreateHandler
from dashboard_handler import DashboardHandler
from manifest_handler import ManifestHandler
@@ -32,10 +35,13 @@
('/admin/report/?', AdminReportHandler),
('/admin/merge-tests/?', MergeTestsHandler),
('/admin/create/(.*)', CreateHandler),
- ('/api/test/?', ManifestHandler),
+ ('/api/test/?', CachedManifestHandler),
+ ('/api/test/update', ManifestHandler),
('/api/test/report/?', ReportHandler),
- ('/api/test/runs/?', RunsHandler),
- ('/api/test/dashboard/?', DashboardHandler),
+ ('/api/test/runs/?', CachedRunsHandler),
+ ('/api/test/runs/update', RunsHandler),
+ ('/api/test/dashboard/?', CachedDashboardHandler),
+ ('/api/test/dashboard/update', DashboardHandler),
]
Modified: trunk/Websites/webkit-perf.appspot.com/manifest_handler.py (107273 => 107274)
--- trunk/Websites/webkit-perf.appspot.com/manifest_handler.py 2012-02-09 20:18:48 UTC (rev 107273)
+++ trunk/Websites/webkit-perf.appspot.com/manifest_handler.py 2012-02-09 20:20:41 UTC (rev 107274)
@@ -28,10 +28,10 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import webapp2
-from google.appengine.api import memcache
import json
+from controller import cache_manifest
from models import Builder
from models import Branch
from models import Platform
@@ -39,12 +39,8 @@
class ManifestHandler(webapp2.RequestHandler):
- def get(self):
- self.response.headers['Content-Type'] = 'text/plain; charset=utf-8';
- cache = memcache.get('manifest')
- if cache:
- self.response.out.write(cache)
- return
+ def post(self):
+ self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
test_map = {}
platform_id_map = {}
@@ -88,6 +84,5 @@
'platformIds': list(set(branch_id_map[branch.id]['platforms'])),
}
- result = json.dumps({'testMap': test_map, 'platformMap': platform_map, 'branchMap': branch_map})
- self.response.out.write(result)
- memcache.add('manifest', result)
+ cache_manifest(json.dumps({'testMap': test_map, 'platformMap': platform_map, 'branchMap': branch_map}))
+ self.response.out.write('OK')
Modified: trunk/Websites/webkit-perf.appspot.com/merge_tests_handler.py (107273 => 107274)
--- trunk/Websites/webkit-perf.appspot.com/merge_tests_handler.py 2012-02-09 20:18:48 UTC (rev 107273)
+++ trunk/Websites/webkit-perf.appspot.com/merge_tests_handler.py 2012-02-09 20:20:41 UTC (rev 107274)
@@ -28,11 +28,13 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import webapp2
-from google.appengine.api import memcache
from google.appengine.ext.webapp import template
import os
+from controller import schedule_runs_update
+from controller import schedule_dashboard_update
+from controller import schedule_manifest_update
from models import Test
from models import TestResult
from models import delete_model_with_numeric_id_holder
@@ -53,14 +55,18 @@
merged_results = TestResult.all()
merged_results.filter('name =', merge.name)
+ branches_and_platforms_to_update = set()
for result in merged_results:
+ branches_and_platforms_to_update.add((result.build.branch.id, result.build.platform.id))
result.name = into.name
result.put()
- # Just flush everyting since we rarely merge tests and we need to flush
- # dashboard, manifest, and all runs for this test here.
- memcache.flush_all()
+ for branch_id, platform_id in branches_and_platforms_to_update:
+ schedule_runs_update(into.id, branch_id, platform_id)
+ schedule_dashboard_update()
+ schedule_manifest_update()
+
delete_model_with_numeric_id_holder(merge)
self.response.out.write('OK')
Modified: trunk/Websites/webkit-perf.appspot.com/report_handler.py (107273 => 107274)
--- trunk/Websites/webkit-perf.appspot.com/report_handler.py 2012-02-09 20:18:48 UTC (rev 107273)
+++ trunk/Websites/webkit-perf.appspot.com/report_handler.py 2012-02-09 20:20:41 UTC (rev 107274)
@@ -28,7 +28,6 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import webapp2
-from google.appengine.api import memcache
from google.appengine.ext import db
import json
@@ -36,6 +35,9 @@
import time
from datetime import datetime
+from controller import schedule_runs_update
+from controller import schedule_dashboard_update
+from controller import schedule_manifest_update
from models import Builder
from models import Branch
from models import Build
@@ -95,7 +97,7 @@
for test_name, result in self._body['results'].iteritems():
test = self._add_test_if_needed(test_name, branch, platform)
- memcache.delete(Test.cache_key(test.id, branch.id, platform.id))
+ schedule_runs_update(test.id, branch.id, platform.id)
if isinstance(result, dict):
TestResult(name=test_name, build=build, value=float(result['avg']), valueMedian=_float_or_none(result, 'median'),
valueStdev=_float_or_none(result, 'stdev'), valueMin=_float_or_none(result, 'min'), valueMax=_float_or_none(result, 'max')).put()
@@ -106,8 +108,8 @@
log.delete()
# We need to update dashboard and manifest because they are affected by the existance of test results
- memcache.delete('dashboard')
- memcache.delete('manifest')
+ schedule_dashboard_update()
+ schedule_manifest_update()
return self._output('OK')
Modified: trunk/Websites/webkit-perf.appspot.com/runs_handler.py (107273 => 107274)
--- trunk/Websites/webkit-perf.appspot.com/runs_handler.py 2012-02-09 20:18:48 UTC (rev 107273)
+++ trunk/Websites/webkit-perf.appspot.com/runs_handler.py 2012-02-09 20:20:41 UTC (rev 107274)
@@ -28,12 +28,12 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import webapp2
-from google.appengine.api import memcache
import json
from time import mktime
from datetime import datetime
+from controller import cache_runs
from models import Build
from models import Builder
from models import Branch
@@ -45,8 +45,8 @@
class RunsHandler(webapp2.RequestHandler):
- def get(self):
- self.response.headers['Content-Type'] = 'application/json; charset=utf-8'
+ def post(self):
+ self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
try:
test_id = int(self.request.get('id', 0))
@@ -61,12 +61,6 @@
# FIXME: Just fetch builds specified by "days"
# days = self.request.get('days', 365)
- cache_key = Test.cache_key(test_id, branch_id, platform_id)
- cache = memcache.get(cache_key)
- if cache:
- self.response.out.write(cache)
- return
-
builds = Build.all()
builds.filter('branch =', modelFromNumericId(branch_id, Branch))
builds.filter('platform =', modelFromNumericId(platform_id, Platform))
@@ -105,5 +99,5 @@
'max': max(values) if values else None,
'date_range': [min(timestamps), max(timestamps)] if timestamps else None,
'stat': 'ok'})
- self.response.out.write(result)
- memcache.add(cache_key, result)
+ cache_runs(test_id, branch_id, platform_id, result)
+ self.response.out.write('OK')
