Diff
Modified: trunk/Tools/ChangeLog (89790 => 89791)
--- trunk/Tools/ChangeLog 2011-06-27 06:50:04 UTC (rev 89790)
+++ trunk/Tools/ChangeLog 2011-06-27 07:10:00 UTC (rev 89791)
@@ -1,3 +1,23 @@
+2011-06-27 Adam Barth <[email protected]>
+
+ Reviewed by Eric Seidel.
+
+ Move deduplicate_tests.py into webkitpy.to_be_moved
+ https://bugs.webkit.org/show_bug.cgi?id=63426
+
+ deduplicate_tests.py doesn't haven anything to do with layout tests.
+ It's really a separate script. This patch moves it to the (new)
+ to_be_moved package so that it doesn't distract us when working on NRWT.
+
+ * Scripts/deduplicate-tests:
+ * Scripts/webkitpy/layout_tests/deduplicate_tests.py: Removed.
+ * Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py: Removed.
+ * Scripts/webkitpy/test/main.py:
+ * Scripts/webkitpy/to_be_moved: Added.
+ * Scripts/webkitpy/to_be_moved/__init__.py: Added.
+ * Scripts/webkitpy/to_be_moved/deduplicate_tests.py: Copied from Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py.
+ * Scripts/webkitpy/to_be_moved/deduplicate_tests_unittest.py: Copied from Tools/Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py.
+
2011-06-26 Adam Barth <[email protected]>
Rubber-stamped by Eric Seidel.
Modified: trunk/Tools/Scripts/deduplicate-tests (89790 => 89791)
--- trunk/Tools/Scripts/deduplicate-tests 2011-06-27 06:50:04 UTC (rev 89790)
+++ trunk/Tools/Scripts/deduplicate-tests 2011-06-27 07:10:00 UTC (rev 89791)
@@ -37,7 +37,7 @@
import optparse
import webkitpy.common.system.logutils as logutils
-import webkitpy.layout_tests.deduplicate_tests as deduplicate_tests
+import webkitpy.to_be_moved.deduplicate_tests as deduplicate_tests
def parse_args():
Deleted: trunk/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py (89790 => 89791)
--- trunk/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py 2011-06-27 06:50:04 UTC (rev 89790)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py 2011-06-27 07:10:00 UTC (rev 89791)
@@ -1,244 +0,0 @@
-#!/usr/bin/env python
-# Copyright (C) 2010 Google Inc. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
-# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
-# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
-# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""deduplicate_tests -- lists duplicated between platforms.
-
-If platform/mac-leopard is missing an expected test output, we fall back on
-platform/mac. This means it's possible to grow redundant test outputs,
-where we have the same expected data in both a platform directory and another
-platform it falls back on.
-"""
-
-import collections
-import fnmatch
-import os
-import subprocess
-import sys
-import re
-
-from webkitpy.common.checkout import scm
-from webkitpy.common.system import executive
-from webkitpy.common.system import logutils
-from webkitpy.common.system import ospath
-from webkitpy.layout_tests.port import factory as port_factory
-
-_log = logutils.get_logger(__file__)
-
-_BASE_PLATFORM = 'base'
-
-
-def port_fallbacks():
- """Get the port fallback information.
- Returns:
- A dictionary mapping platform name to a list of other platforms to fall
- back on. All platforms fall back on 'base'.
- """
- fallbacks = {_BASE_PLATFORM: []}
- for port_name in port_factory.all_port_names():
- try:
- platforms = port_factory.get(port_name).baseline_search_path()
- except NotImplementedError:
- _log.error("'%s' lacks baseline_search_path(), please fix."
- % port_name)
- fallbacks[port_name] = [_BASE_PLATFORM]
- continue
- fallbacks[port_name] = [os.path.basename(p) for p in platforms][1:]
- fallbacks[port_name].append(_BASE_PLATFORM)
-
- return fallbacks
-
-
-def parse_git_output(git_output, glob_pattern):
- """Parses the output of git ls-tree and filters based on glob_pattern.
- Args:
- git_output: result of git ls-tree -r HEAD LayoutTests.
- glob_pattern: a pattern to filter the files.
- Returns:
- A dictionary mapping (test name, hash of content) => [paths]
- """
- hashes = collections.defaultdict(set)
- for line in git_output.split('\n'):
- if not line:
- break
- attrs, path = line.strip().split('\t')
- if not fnmatch.fnmatch(path, glob_pattern):
- continue
- path = path[len('LayoutTests/'):]
- match = re.match(r'^(platform/.*?/)?(.*)', path)
- test = match.group(2)
- _, _, hash = attrs.split(' ')
- hashes[(test, hash)].add(path)
- return hashes
-
-
-def cluster_file_hashes(glob_pattern):
- """Get the hashes of all the test expectations in the tree.
- We cheat and use git's hashes.
- Args:
- glob_pattern: a pattern to filter the files.
- Returns:
- A dictionary mapping (test name, hash of content) => [paths]
- """
-
- # A map of file hash => set of all files with that hash.
- hashes = collections.defaultdict(set)
-
- # Fill in the map.
- cmd = ('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests')
- try:
- git_output = executive.Executive().run_command(cmd,
- cwd=scm.find_checkout_root())
- except OSError, e:
- if e.errno == 2: # No such file or directory.
- _log.error("Error: 'No such file' when running git.")
- _log.error("This script requires git.")
- sys.exit(1)
- raise e
- return parse_git_output(git_output, glob_pattern)
-
-
-def dirname_to_platform(dirname):
- if dirname == 'chromium-linux':
- return 'chromium-linux-x86_64'
- elif dirname == 'chromium-win':
- return 'chromium-win-win7'
- elif dirname == 'chromium-mac':
- return 'chromium-mac-snowleopard'
- return dirname
-
-def extract_platforms(paths):
- """Extracts the platforms from a list of paths matching ^platform/(.*?)/.
- Args:
- paths: a list of paths.
- Returns:
- A dictionary containing all platforms from paths.
- """
- platforms = {}
- for path in paths:
- match = re.match(r'^platform/(.*?)/', path)
- if match:
- platform = dirname_to_platform(match.group(1))
- else:
- platform = _BASE_PLATFORM
- platforms[platform] = path
- return platforms
-
-
-def has_intermediate_results(test, fallbacks, matching_platform,
- path_exists=os.path.exists):
- """Returns True if there is a test result that causes us to not delete
- this duplicate.
-
- For example, chromium-linux may be a duplicate of the checked in result,
- but chromium-win may have a different result checked in. In this case,
- we need to keep the duplicate results.
-
- Args:
- test: The test name.
- fallbacks: A list of platforms we fall back on.
- matching_platform: The platform that we found the duplicate test
- result. We can stop checking here.
- path_exists: Optional parameter that allows us to stub out
- os.path.exists for testing.
- """
- for dirname in fallbacks:
- platform = dirname_to_platform(dirname)
- if platform == matching_platform:
- return False
- test_path = os.path.join('LayoutTests', 'platform', dirname, test)
- if path_exists(test_path):
- return True
- return False
-
-
-def get_relative_test_path(filename, relative_to,
- checkout_root=scm.find_checkout_root()):
- """Constructs a relative path to |filename| from |relative_to|.
- Args:
- filename: The test file we're trying to get a relative path to.
- relative_to: The absolute path we're relative to.
- Returns:
- A relative path to filename or None if |filename| is not below
- |relative_to|.
- """
- layout_test_dir = os.path.join(checkout_root, 'LayoutTests')
- abs_path = os.path.join(layout_test_dir, filename)
- return ospath.relpath(abs_path, relative_to)
-
-
-def find_dups(hashes, port_fallbacks, relative_to):
- """Yields info about redundant test expectations.
- Args:
- hashes: a list of hashes as returned by cluster_file_hashes.
- port_fallbacks: a list of fallback information as returned by
- get_port_fallbacks.
- relative_to: the directory that we want the results relative to
- Returns:
- a tuple containing (test, platform, fallback, platforms)
- """
- for (test, hash), cluster in hashes.items():
- if len(cluster) < 2:
- continue # Common case: only one file with that hash.
-
- # Compute the list of platforms we have this particular hash for.
- platforms = extract_platforms(cluster)
- if len(platforms) == 1:
- continue
-
- # See if any of the platforms are redundant with each other.
- for platform in platforms.keys():
- if platform not in port_factory.all_port_names():
- continue
- for dirname in port_fallbacks[platform]:
- fallback = dirname_to_platform(dirname)
- if fallback not in platforms.keys():
- continue
- # We have to verify that there isn't an intermediate result
- # that causes this duplicate hash to exist.
- if has_intermediate_results(test, port_fallbacks[platform],
- fallback):
- continue
- # We print the relative path so it's easy to pipe the results
- # to xargs rm.
- path = get_relative_test_path(platforms[platform], relative_to)
- if not path:
- continue
- yield {
- 'test': test,
- 'platform': platform,
- 'fallback': dirname,
- 'path': path,
- }
-
-
-def deduplicate(glob_pattern):
- """Traverses LayoutTests and returns information about duplicated files.
- Args:
- glob pattern to filter the files in LayoutTests.
- Returns:
- a dictionary containing test, path, platform and fallback.
- """
- fallbacks = port_fallbacks()
- hashes = cluster_file_hashes(glob_pattern)
- return list(find_dups(hashes, fallbacks, os.getcwd()))
Deleted: trunk/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py (89790 => 89791)
--- trunk/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py 2011-06-27 06:50:04 UTC (rev 89790)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py 2011-06-27 07:10:00 UTC (rev 89791)
@@ -1,210 +0,0 @@
-#!/usr/bin/env python
-# Copyright (C) 2010 Google Inc. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
-# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
-# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
-# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""Unit tests for deduplicate_tests.py."""
-
-import deduplicate_tests
-import os
-import unittest
-import webkitpy.common.checkout.scm as scm
-
-
-class MockExecutive(object):
- last_run_command = []
- response = ''
-
- class Executive(object):
- def run_command(self,
- args,
- cwd=None,
- input=None,
- error_handler=None,
- return_exit_code=False,
- return_stderr=True,
- decode_output=True):
- MockExecutive.last_run_command += [args]
- return MockExecutive.response
-
-
-class ListDuplicatesTest(unittest.TestCase):
- def setUp(self):
- MockExecutive.last_run_command = []
- MockExecutive.response = ''
- deduplicate_tests.executive = MockExecutive
- self._original_cwd = os.getcwd()
- checkout_root = scm.find_checkout_root()
- self.assertNotEqual(checkout_root, None)
- os.chdir(checkout_root)
-
- def tearDown(self):
- os.chdir(self._original_cwd)
-
- def test_parse_git_output(self):
- git_output = (
- '100644 blob 5053240b3353f6eb39f7cb00259785f16d121df2\tLayoutTests/mac/foo-expected.txt\n'
- '100644 blob a004548d107ecc4e1ea08019daf0a14e8634a1ff\tLayoutTests/platform/chromium/foo-expected.txt\n'
- '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/foo-expected.txt\n'
- '100644 blob abcdebc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/animage.png\n'
- '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/foo-expected.txt\n'
- '100644 blob abcdebc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/animage.png\n'
- '100644 blob 4303df5389ca87cae83dd3236b8dd84e16606517\tLayoutTests/platform/mac/foo-expected.txt\n')
- hashes = deduplicate_tests.parse_git_output(git_output, '*')
- expected = {('mac/foo-expected.txt', '5053240b3353f6eb39f7cb00259785f16d121df2'): set(['mac/foo-expected.txt']),
- ('animage.png', 'abcdebc762e3aec5df03b5c04485b2cb3b65ffb1'): set(['platform/chromium-linux/animage.png', 'platform/chromium-win/animage.png']),
- ('foo-expected.txt', '4303df5389ca87cae83dd3236b8dd84e16606517'): set(['platform/mac/foo-expected.txt']),
- ('foo-expected.txt', 'd6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1'): set(['platform/chromium-linux/foo-expected.txt', 'platform/chromium-win/foo-expected.txt']),
- ('foo-expected.txt', 'a004548d107ecc4e1ea08019daf0a14e8634a1ff'): set(['platform/chromium/foo-expected.txt'])}
- self.assertEquals(expected, hashes)
-
- hashes = deduplicate_tests.parse_git_output(git_output, '*.png')
- expected = {('animage.png', 'abcdebc762e3aec5df03b5c04485b2cb3b65ffb1'): set(['platform/chromium-linux/animage.png', 'platform/chromium-win/animage.png'])}
- self.assertEquals(expected, hashes)
-
- def test_extract_platforms(self):
- self.assertEquals({'foo': 'platform/foo/bar',
- 'zoo': 'platform/zoo/com'},
- deduplicate_tests.extract_platforms(['platform/foo/bar', 'platform/zoo/com']))
- self.assertEquals({'foo': 'platform/foo/bar',
- deduplicate_tests._BASE_PLATFORM: 'what/'},
- deduplicate_tests.extract_platforms(['platform/foo/bar', 'what/']))
-
- def test_has_intermediate_results(self):
- test_cases = (
- # If we found a duplicate in our first fallback, we have no
- # intermediate results.
- (False, ('fast/foo-expected.txt',
- ['chromium-win', 'chromium', 'base'],
- 'chromium-win-win7',
- lambda path: True)),
- # Since chromium-win has a result, we have an intermediate result.
- (True, ('fast/foo-expected.txt',
- ['chromium-win', 'chromium', 'base'],
- 'chromium',
- lambda path: True)),
- # There are no intermediate results.
- (False, ('fast/foo-expected.txt',
- ['chromium-win', 'chromium', 'base'],
- 'chromium',
- lambda path: False)),
- # There are no intermediate results since a result for chromium is
- # our duplicate file.
- (False, ('fast/foo-expected.txt',
- ['chromium-win', 'chromium', 'base'],
- 'chromium',
- lambda path: path == 'LayoutTests/platform/chromium/fast/foo-expected.txt')),
- # We have an intermediate result in 'chromium' even though our
- # duplicate is with the file in 'base'.
- (True, ('fast/foo-expected.txt',
- ['chromium-win', 'chromium', 'base'],
- 'base',
- lambda path: path == 'LayoutTests/platform/chromium/fast/foo-expected.txt')),
- # We have an intermediate result in 'chromium-win' even though our
- # duplicate is in 'base'.
- (True, ('fast/foo-expected.txt',
- ['chromium-win', 'chromium', 'base'],
- 'base',
- lambda path: path == 'LayoutTests/platform/chromium-win/fast/foo-expected.txt')),
- )
- for expected, inputs in test_cases:
- self.assertEquals(expected,
- deduplicate_tests.has_intermediate_results(*inputs))
-
- def test_unique(self):
- MockExecutive.response = (
- '100644 blob 5053240b3353f6eb39f7cb00259785f16d121df2\tLayoutTests/mac/foo-expected.txt\n'
- '100644 blob a004548d107ecc4e1ea08019daf0a14e8634a1ff\tLayoutTests/platform/chromium/foo-expected.txt\n'
- '100644 blob abcd0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/foo-expected.txt\n'
- '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/foo-expected.txt\n'
- '100644 blob 4303df5389ca87cae83dd3236b8dd84e16606517\tLayoutTests/platform/mac/foo-expected.txt\n')
- result = deduplicate_tests.deduplicate('*')
- self.assertEquals(1, len(MockExecutive.last_run_command))
- self.assertEquals(('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests'), MockExecutive.last_run_command[-1])
- self.assertEquals(0, len(result))
-
- def test_duplicates(self):
- MockExecutive.response = (
- '100644 blob 5053240b3353f6eb39f7cb00259785f16d121df2\tLayoutTests/mac/foo-expected.txt\n'
- '100644 blob a004548d107ecc4e1ea08019daf0a14e8634a1ff\tLayoutTests/platform/chromium/foo-expected.txt\n'
- '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/foo-expected.txt\n'
- '100644 blob abcdebc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/animage.png\n'
- '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/foo-expected.txt\n'
- '100644 blob abcdebc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/animage.png\n'
- '100644 blob 4303df5389ca87cae83dd3236b8dd84e16606517\tLayoutTests/platform/mac/foo-expected.txt\n')
-
- result = deduplicate_tests.deduplicate('*')
- self.assertEquals(1, len(MockExecutive.last_run_command))
- self.assertEquals(('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests'), MockExecutive.last_run_command[-1])
- self.assertEquals(2, len(result))
- self.assertEquals({'test': 'animage.png',
- 'path': 'LayoutTests/platform/chromium-linux/animage.png',
- 'fallback': 'chromium-win',
- 'platform': 'chromium-linux-x86_64'},
- result[0])
- self.assertEquals({'test': 'foo-expected.txt',
- 'path': 'LayoutTests/platform/chromium-linux/foo-expected.txt',
- 'fallback': 'chromium-win',
- 'platform': 'chromium-linux-x86_64'},
- result[1])
-
- result = deduplicate_tests.deduplicate('*.txt')
- self.assertEquals(2, len(MockExecutive.last_run_command))
- self.assertEquals(('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests'), MockExecutive.last_run_command[-1])
- self.assertEquals(1, len(result))
- self.assertEquals({'test': 'foo-expected.txt',
- 'path': 'LayoutTests/platform/chromium-linux/foo-expected.txt',
- 'fallback': 'chromium-win',
- 'platform': 'chromium-linux-x86_64'},
- result[0])
-
- result = deduplicate_tests.deduplicate('*.png')
- self.assertEquals(3, len(MockExecutive.last_run_command))
- self.assertEquals(('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests'), MockExecutive.last_run_command[-1])
- self.assertEquals(1, len(result))
- self.assertEquals({'test': 'animage.png',
- 'path': 'LayoutTests/platform/chromium-linux/animage.png',
- 'fallback': 'chromium-win',
- 'platform': 'chromium-linux-x86_64'},
- result[0])
-
- def test_get_relative_test_path(self):
- checkout_root = scm.find_checkout_root()
- layout_test_dir = os.path.join(checkout_root, 'LayoutTests')
- test_cases = (
- ('platform/mac/test.html',
- ('platform/mac/test.html', layout_test_dir)),
- ('LayoutTests/platform/mac/test.html',
- ('platform/mac/test.html', checkout_root)),
- (None,
- ('platform/mac/test.html', os.path.join(checkout_root, 'Source', 'WebCore'))),
- ('test.html',
- ('platform/mac/test.html', os.path.join(layout_test_dir, 'platform/mac'))),
- (None,
- ('platform/mac/test.html', os.path.join(layout_test_dir, 'platform/win'))),
- )
- for expected, inputs in test_cases:
- self.assertEquals(expected,
- deduplicate_tests.get_relative_test_path(*inputs))
-
-if __name__ == '__main__':
- unittest.main()
Modified: trunk/Tools/Scripts/webkitpy/test/main.py (89790 => 89791)
--- trunk/Tools/Scripts/webkitpy/test/main.py 2011-06-27 06:50:04 UTC (rev 89790)
+++ trunk/Tools/Scripts/webkitpy/test/main.py 2011-06-27 07:10:00 UTC (rev 89791)
@@ -90,7 +90,7 @@
return module_path not in [
# FIXME: This file also requires common.checkout to work
- 'webkitpy.layout_tests.deduplicate_tests_unittest',
+ 'webkitpy.to_be_moved.deduplicate_tests_unittest',
]
def run_tests(self, sys_argv, external_package_paths=None):
Added: trunk/Tools/Scripts/webkitpy/to_be_moved/__init__.py (0 => 89791)
--- trunk/Tools/Scripts/webkitpy/to_be_moved/__init__.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/to_be_moved/__init__.py 2011-06-27 07:10:00 UTC (rev 89791)
@@ -0,0 +1,10 @@
+# Required for Python to search this directory for module files
+# This directory houses Python modules that do not yet have a proper home.
+#
+# Some of the Python modules in this directory aren't really part of webkitpy
+# in the sense that they're not classes that are meant to be used as part of
+# the webkitpy library. Instead, they're a bunch of helper code for individual
+# scripts in in Tools/Scripts.
+#
+# Really, all this code should either be refactored or moved somewhere else,
+# hence the somewhat lame name for this directory.
Copied: trunk/Tools/Scripts/webkitpy/to_be_moved/deduplicate_tests.py (from rev 89785, trunk/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py) (0 => 89791)
--- trunk/Tools/Scripts/webkitpy/to_be_moved/deduplicate_tests.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/to_be_moved/deduplicate_tests.py 2011-06-27 07:10:00 UTC (rev 89791)
@@ -0,0 +1,244 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""deduplicate_tests -- lists duplicated between platforms.
+
+If platform/mac-leopard is missing an expected test output, we fall back on
+platform/mac. This means it's possible to grow redundant test outputs,
+where we have the same expected data in both a platform directory and another
+platform it falls back on.
+"""
+
+import collections
+import fnmatch
+import os
+import subprocess
+import sys
+import re
+
+from webkitpy.common.checkout import scm
+from webkitpy.common.system import executive
+from webkitpy.common.system import logutils
+from webkitpy.common.system import ospath
+from webkitpy.layout_tests.port import factory as port_factory
+
+_log = logutils.get_logger(__file__)
+
+_BASE_PLATFORM = 'base'
+
+
+def port_fallbacks():
+ """Get the port fallback information.
+ Returns:
+ A dictionary mapping platform name to a list of other platforms to fall
+ back on. All platforms fall back on 'base'.
+ """
+ fallbacks = {_BASE_PLATFORM: []}
+ for port_name in port_factory.all_port_names():
+ try:
+ platforms = port_factory.get(port_name).baseline_search_path()
+ except NotImplementedError:
+ _log.error("'%s' lacks baseline_search_path(), please fix."
+ % port_name)
+ fallbacks[port_name] = [_BASE_PLATFORM]
+ continue
+ fallbacks[port_name] = [os.path.basename(p) for p in platforms][1:]
+ fallbacks[port_name].append(_BASE_PLATFORM)
+
+ return fallbacks
+
+
+def parse_git_output(git_output, glob_pattern):
+ """Parses the output of git ls-tree and filters based on glob_pattern.
+ Args:
+ git_output: result of git ls-tree -r HEAD LayoutTests.
+ glob_pattern: a pattern to filter the files.
+ Returns:
+ A dictionary mapping (test name, hash of content) => [paths]
+ """
+ hashes = collections.defaultdict(set)
+ for line in git_output.split('\n'):
+ if not line:
+ break
+ attrs, path = line.strip().split('\t')
+ if not fnmatch.fnmatch(path, glob_pattern):
+ continue
+ path = path[len('LayoutTests/'):]
+ match = re.match(r'^(platform/.*?/)?(.*)', path)
+ test = match.group(2)
+ _, _, hash = attrs.split(' ')
+ hashes[(test, hash)].add(path)
+ return hashes
+
+
+def cluster_file_hashes(glob_pattern):
+ """Get the hashes of all the test expectations in the tree.
+ We cheat and use git's hashes.
+ Args:
+ glob_pattern: a pattern to filter the files.
+ Returns:
+ A dictionary mapping (test name, hash of content) => [paths]
+ """
+
+ # A map of file hash => set of all files with that hash.
+ hashes = collections.defaultdict(set)
+
+ # Fill in the map.
+ cmd = ('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests')
+ try:
+ git_output = executive.Executive().run_command(cmd,
+ cwd=scm.find_checkout_root())
+ except OSError, e:
+ if e.errno == 2: # No such file or directory.
+ _log.error("Error: 'No such file' when running git.")
+ _log.error("This script requires git.")
+ sys.exit(1)
+ raise e
+ return parse_git_output(git_output, glob_pattern)
+
+
+def dirname_to_platform(dirname):
+ if dirname == 'chromium-linux':
+ return 'chromium-linux-x86_64'
+ elif dirname == 'chromium-win':
+ return 'chromium-win-win7'
+ elif dirname == 'chromium-mac':
+ return 'chromium-mac-snowleopard'
+ return dirname
+
+def extract_platforms(paths):
+ """Extracts the platforms from a list of paths matching ^platform/(.*?)/.
+ Args:
+ paths: a list of paths.
+ Returns:
+ A dictionary containing all platforms from paths.
+ """
+ platforms = {}
+ for path in paths:
+ match = re.match(r'^platform/(.*?)/', path)
+ if match:
+ platform = dirname_to_platform(match.group(1))
+ else:
+ platform = _BASE_PLATFORM
+ platforms[platform] = path
+ return platforms
+
+
+def has_intermediate_results(test, fallbacks, matching_platform,
+ path_exists=os.path.exists):
+ """Returns True if there is a test result that causes us to not delete
+ this duplicate.
+
+ For example, chromium-linux may be a duplicate of the checked in result,
+ but chromium-win may have a different result checked in. In this case,
+ we need to keep the duplicate results.
+
+ Args:
+ test: The test name.
+ fallbacks: A list of platforms we fall back on.
+ matching_platform: The platform that we found the duplicate test
+ result. We can stop checking here.
+ path_exists: Optional parameter that allows us to stub out
+ os.path.exists for testing.
+ """
+ for dirname in fallbacks:
+ platform = dirname_to_platform(dirname)
+ if platform == matching_platform:
+ return False
+ test_path = os.path.join('LayoutTests', 'platform', dirname, test)
+ if path_exists(test_path):
+ return True
+ return False
+
+
+def get_relative_test_path(filename, relative_to,
+ checkout_root=scm.find_checkout_root()):
+ """Constructs a relative path to |filename| from |relative_to|.
+ Args:
+ filename: The test file we're trying to get a relative path to.
+ relative_to: The absolute path we're relative to.
+ Returns:
+ A relative path to filename or None if |filename| is not below
+ |relative_to|.
+ """
+ layout_test_dir = os.path.join(checkout_root, 'LayoutTests')
+ abs_path = os.path.join(layout_test_dir, filename)
+ return ospath.relpath(abs_path, relative_to)
+
+
+def find_dups(hashes, port_fallbacks, relative_to):
+ """Yields info about redundant test expectations.
+ Args:
+ hashes: a list of hashes as returned by cluster_file_hashes.
+ port_fallbacks: a list of fallback information as returned by
+ get_port_fallbacks.
+ relative_to: the directory that we want the results relative to
+ Returns:
+ a tuple containing (test, platform, fallback, platforms)
+ """
+ for (test, hash), cluster in hashes.items():
+ if len(cluster) < 2:
+ continue # Common case: only one file with that hash.
+
+ # Compute the list of platforms we have this particular hash for.
+ platforms = extract_platforms(cluster)
+ if len(platforms) == 1:
+ continue
+
+ # See if any of the platforms are redundant with each other.
+ for platform in platforms.keys():
+ if platform not in port_factory.all_port_names():
+ continue
+ for dirname in port_fallbacks[platform]:
+ fallback = dirname_to_platform(dirname)
+ if fallback not in platforms.keys():
+ continue
+ # We have to verify that there isn't an intermediate result
+ # that causes this duplicate hash to exist.
+ if has_intermediate_results(test, port_fallbacks[platform],
+ fallback):
+ continue
+ # We print the relative path so it's easy to pipe the results
+ # to xargs rm.
+ path = get_relative_test_path(platforms[platform], relative_to)
+ if not path:
+ continue
+ yield {
+ 'test': test,
+ 'platform': platform,
+ 'fallback': dirname,
+ 'path': path,
+ }
+
+
+def deduplicate(glob_pattern):
+ """Traverses LayoutTests and returns information about duplicated files.
+ Args:
+ glob pattern to filter the files in LayoutTests.
+ Returns:
+ a dictionary containing test, path, platform and fallback.
+ """
+ fallbacks = port_fallbacks()
+ hashes = cluster_file_hashes(glob_pattern)
+ return list(find_dups(hashes, fallbacks, os.getcwd()))
Copied: trunk/Tools/Scripts/webkitpy/to_be_moved/deduplicate_tests_unittest.py (from rev 89785, trunk/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py) (0 => 89791)
--- trunk/Tools/Scripts/webkitpy/to_be_moved/deduplicate_tests_unittest.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/to_be_moved/deduplicate_tests_unittest.py 2011-06-27 07:10:00 UTC (rev 89791)
@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit tests for deduplicate_tests.py."""
+
+import deduplicate_tests
+import os
+import unittest
+import webkitpy.common.checkout.scm as scm
+
+
+class MockExecutive(object):
+ last_run_command = []
+ response = ''
+
+ class Executive(object):
+ def run_command(self,
+ args,
+ cwd=None,
+ input=None,
+ error_handler=None,
+ return_exit_code=False,
+ return_stderr=True,
+ decode_output=True):
+ MockExecutive.last_run_command += [args]
+ return MockExecutive.response
+
+
+class ListDuplicatesTest(unittest.TestCase):
+ def setUp(self):
+ MockExecutive.last_run_command = []
+ MockExecutive.response = ''
+ deduplicate_tests.executive = MockExecutive
+ self._original_cwd = os.getcwd()
+ checkout_root = scm.find_checkout_root()
+ self.assertNotEqual(checkout_root, None)
+ os.chdir(checkout_root)
+
+ def tearDown(self):
+ os.chdir(self._original_cwd)
+
+ def test_parse_git_output(self):
+ git_output = (
+ '100644 blob 5053240b3353f6eb39f7cb00259785f16d121df2\tLayoutTests/mac/foo-expected.txt\n'
+ '100644 blob a004548d107ecc4e1ea08019daf0a14e8634a1ff\tLayoutTests/platform/chromium/foo-expected.txt\n'
+ '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/foo-expected.txt\n'
+ '100644 blob abcdebc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/animage.png\n'
+ '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/foo-expected.txt\n'
+ '100644 blob abcdebc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/animage.png\n'
+ '100644 blob 4303df5389ca87cae83dd3236b8dd84e16606517\tLayoutTests/platform/mac/foo-expected.txt\n')
+ hashes = deduplicate_tests.parse_git_output(git_output, '*')
+ expected = {('mac/foo-expected.txt', '5053240b3353f6eb39f7cb00259785f16d121df2'): set(['mac/foo-expected.txt']),
+ ('animage.png', 'abcdebc762e3aec5df03b5c04485b2cb3b65ffb1'): set(['platform/chromium-linux/animage.png', 'platform/chromium-win/animage.png']),
+ ('foo-expected.txt', '4303df5389ca87cae83dd3236b8dd84e16606517'): set(['platform/mac/foo-expected.txt']),
+ ('foo-expected.txt', 'd6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1'): set(['platform/chromium-linux/foo-expected.txt', 'platform/chromium-win/foo-expected.txt']),
+ ('foo-expected.txt', 'a004548d107ecc4e1ea08019daf0a14e8634a1ff'): set(['platform/chromium/foo-expected.txt'])}
+ self.assertEquals(expected, hashes)
+
+ hashes = deduplicate_tests.parse_git_output(git_output, '*.png')
+ expected = {('animage.png', 'abcdebc762e3aec5df03b5c04485b2cb3b65ffb1'): set(['platform/chromium-linux/animage.png', 'platform/chromium-win/animage.png'])}
+ self.assertEquals(expected, hashes)
+
+ def test_extract_platforms(self):
+ self.assertEquals({'foo': 'platform/foo/bar',
+ 'zoo': 'platform/zoo/com'},
+ deduplicate_tests.extract_platforms(['platform/foo/bar', 'platform/zoo/com']))
+ self.assertEquals({'foo': 'platform/foo/bar',
+ deduplicate_tests._BASE_PLATFORM: 'what/'},
+ deduplicate_tests.extract_platforms(['platform/foo/bar', 'what/']))
+
+ def test_has_intermediate_results(self):
+ test_cases = (
+ # If we found a duplicate in our first fallback, we have no
+ # intermediate results.
+ (False, ('fast/foo-expected.txt',
+ ['chromium-win', 'chromium', 'base'],
+ 'chromium-win-win7',
+ lambda path: True)),
+ # Since chromium-win has a result, we have an intermediate result.
+ (True, ('fast/foo-expected.txt',
+ ['chromium-win', 'chromium', 'base'],
+ 'chromium',
+ lambda path: True)),
+ # There are no intermediate results.
+ (False, ('fast/foo-expected.txt',
+ ['chromium-win', 'chromium', 'base'],
+ 'chromium',
+ lambda path: False)),
+ # There are no intermediate results since a result for chromium is
+ # our duplicate file.
+ (False, ('fast/foo-expected.txt',
+ ['chromium-win', 'chromium', 'base'],
+ 'chromium',
+ lambda path: path == 'LayoutTests/platform/chromium/fast/foo-expected.txt')),
+ # We have an intermediate result in 'chromium' even though our
+ # duplicate is with the file in 'base'.
+ (True, ('fast/foo-expected.txt',
+ ['chromium-win', 'chromium', 'base'],
+ 'base',
+ lambda path: path == 'LayoutTests/platform/chromium/fast/foo-expected.txt')),
+ # We have an intermediate result in 'chromium-win' even though our
+ # duplicate is in 'base'.
+ (True, ('fast/foo-expected.txt',
+ ['chromium-win', 'chromium', 'base'],
+ 'base',
+ lambda path: path == 'LayoutTests/platform/chromium-win/fast/foo-expected.txt')),
+ )
+ for expected, inputs in test_cases:
+ self.assertEquals(expected,
+ deduplicate_tests.has_intermediate_results(*inputs))
+
+ def test_unique(self):
+ MockExecutive.response = (
+ '100644 blob 5053240b3353f6eb39f7cb00259785f16d121df2\tLayoutTests/mac/foo-expected.txt\n'
+ '100644 blob a004548d107ecc4e1ea08019daf0a14e8634a1ff\tLayoutTests/platform/chromium/foo-expected.txt\n'
+ '100644 blob abcd0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/foo-expected.txt\n'
+ '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/foo-expected.txt\n'
+ '100644 blob 4303df5389ca87cae83dd3236b8dd84e16606517\tLayoutTests/platform/mac/foo-expected.txt\n')
+ result = deduplicate_tests.deduplicate('*')
+ self.assertEquals(1, len(MockExecutive.last_run_command))
+ self.assertEquals(('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests'), MockExecutive.last_run_command[-1])
+ self.assertEquals(0, len(result))
+
+ def test_duplicates(self):
+ MockExecutive.response = (
+ '100644 blob 5053240b3353f6eb39f7cb00259785f16d121df2\tLayoutTests/mac/foo-expected.txt\n'
+ '100644 blob a004548d107ecc4e1ea08019daf0a14e8634a1ff\tLayoutTests/platform/chromium/foo-expected.txt\n'
+ '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/foo-expected.txt\n'
+ '100644 blob abcdebc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/animage.png\n'
+ '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/foo-expected.txt\n'
+ '100644 blob abcdebc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/animage.png\n'
+ '100644 blob 4303df5389ca87cae83dd3236b8dd84e16606517\tLayoutTests/platform/mac/foo-expected.txt\n')
+
+ result = deduplicate_tests.deduplicate('*')
+ self.assertEquals(1, len(MockExecutive.last_run_command))
+ self.assertEquals(('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests'), MockExecutive.last_run_command[-1])
+ self.assertEquals(2, len(result))
+ self.assertEquals({'test': 'animage.png',
+ 'path': 'LayoutTests/platform/chromium-linux/animage.png',
+ 'fallback': 'chromium-win',
+ 'platform': 'chromium-linux-x86_64'},
+ result[0])
+ self.assertEquals({'test': 'foo-expected.txt',
+ 'path': 'LayoutTests/platform/chromium-linux/foo-expected.txt',
+ 'fallback': 'chromium-win',
+ 'platform': 'chromium-linux-x86_64'},
+ result[1])
+
+ result = deduplicate_tests.deduplicate('*.txt')
+ self.assertEquals(2, len(MockExecutive.last_run_command))
+ self.assertEquals(('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests'), MockExecutive.last_run_command[-1])
+ self.assertEquals(1, len(result))
+ self.assertEquals({'test': 'foo-expected.txt',
+ 'path': 'LayoutTests/platform/chromium-linux/foo-expected.txt',
+ 'fallback': 'chromium-win',
+ 'platform': 'chromium-linux-x86_64'},
+ result[0])
+
+ result = deduplicate_tests.deduplicate('*.png')
+ self.assertEquals(3, len(MockExecutive.last_run_command))
+ self.assertEquals(('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests'), MockExecutive.last_run_command[-1])
+ self.assertEquals(1, len(result))
+ self.assertEquals({'test': 'animage.png',
+ 'path': 'LayoutTests/platform/chromium-linux/animage.png',
+ 'fallback': 'chromium-win',
+ 'platform': 'chromium-linux-x86_64'},
+ result[0])
+
+ def test_get_relative_test_path(self):
+ checkout_root = scm.find_checkout_root()
+ layout_test_dir = os.path.join(checkout_root, 'LayoutTests')
+ test_cases = (
+ ('platform/mac/test.html',
+ ('platform/mac/test.html', layout_test_dir)),
+ ('LayoutTests/platform/mac/test.html',
+ ('platform/mac/test.html', checkout_root)),
+ (None,
+ ('platform/mac/test.html', os.path.join(checkout_root, 'Source', 'WebCore'))),
+ ('test.html',
+ ('platform/mac/test.html', os.path.join(layout_test_dir, 'platform/mac'))),
+ (None,
+ ('platform/mac/test.html', os.path.join(layout_test_dir, 'platform/win'))),
+ )
+ for expected, inputs in test_cases:
+ self.assertEquals(expected,
+ deduplicate_tests.get_relative_test_path(*inputs))
+
+if __name__ == '__main__':
+ unittest.main()
