Diff
Modified: trunk/Tools/ChangeLog (277101 => 277102)
--- trunk/Tools/ChangeLog 2021-05-06 17:20:55 UTC (rev 277101)
+++ trunk/Tools/ChangeLog 2021-05-06 18:32:40 UTC (rev 277102)
@@ -1,3 +1,61 @@
+2021-05-06 Jonathan Bedard <jbed...@apple.com>
+
+ [webkitcorepy] Add API to efficiently create a sequence of commits
+ https://bugs.webkit.org/show_bug.cgi?id=224890
+ <rdar://problem/76975733>
+
+ Rubber-stamped by Aakash Jain.
+
+ While it is possible to simple iterate through a range of commits to define them,
+ every API we use to define commits has much more efficient techniques.
+
+ * Scripts/libraries/webkitscmpy/setup.py: Bump version.
+ * Scripts/libraries/webkitscmpy/webkitscmpy/__init__.py: Ditto.
+ * Scripts/libraries/webkitscmpy/webkitscmpy/contributor.py:
+ (Contributor): Add revision to SVN_AUTHOR_RE and add regex without lines.
+ (Contributor.from_scm_log): Strip leading whitespace from author.
+ * Scripts/libraries/webkitscmpy/webkitscmpy/local/git.py:
+ (Git._args_from_content):
+ (Git.commits): Use `git log` to efficiently compute a range of commits.
+ * Scripts/libraries/webkitscmpy/webkitscmpy/local/svn.py:
+ (Svn._args_from_content):
+ (Svn.commits): Use `svn log` to efficiently compute a range of commits.
+ * Scripts/libraries/webkitscmpy/webkitscmpy/mocks/local/git.py:
+ (Git.__init__): Add `git log` mock.
+ * Scripts/libraries/webkitscmpy/webkitscmpy/mocks/local/svn.py:
+ (Svn.__init__): Add `svn log` mock and more explicit `svn info` mock.
+ (Svn._log_range):
+ * Scripts/libraries/webkitscmpy/webkitscmpy/mocks/remote/git_hub.py:
+ (GitHub._commits_response): Return all parent commits to provided ref.
+ (GitHub.request):
+ * Scripts/libraries/webkitscmpy/webkitscmpy/mocks/remote/svn.py:
+ (Svn.range): More efficiently compute the range.
+ (Svn.request):
+ * Scripts/libraries/webkitscmpy/webkitscmpy/remote/git_hub.py:
+ (GitHub.request): Allow caller to disable pagination.
+ (GitHub.commit): Reduce number of requests required to compute order.
+ (GitHub.commits): Using the `commits` endpoint, more efficiently
+ compute a range of commits.
+ * Scripts/libraries/webkitscmpy/webkitscmpy/remote/svn.py:
+ (Svn): Generalize HISTORY_RE to match any single-line SVN XML response.
+ (Svn._cache_revisions): Replace HISTORY_RE with DATA_RE.
+ (Svn.commits): Use svn/rvr to efficiently compute a range of commits.
+ * Scripts/libraries/webkitscmpy/webkitscmpy/scm_base.py:
+ (ScmBase._commit_range): Return a pair of commits representing the range
+ the caller is requesting, and preform some basic sanity checks.
+ (ScmBase.commits): Declare function implemented by decedents.
+ * Scripts/libraries/webkitscmpy/webkitscmpy/test/find_unittest.py:
+ * Scripts/libraries/webkitscmpy/webkitscmpy/test/git_unittest.py:
+ (TestGit.test_commits):
+ (TestGit.test_commits_branch):
+ (TestGitHub.test_commits):
+ (TestGitHub.test_commits_branch):
+ * Scripts/libraries/webkitscmpy/webkitscmpy/test/svn_unittest.py:
+ (TestLocalSvn.test_commits):
+ (TestLocalSvn.test_commits_branch):
+ (TestRemoteSvn.test_commits):
+ (TestRemoteSvn.test_commits_branch):
+
2021-05-06 Chris Dumez <cdu...@apple.com>
REGRESSION (r272414?): [macOS] TestWebKitAPI.GPUProcess.CrashWhilePlayingVideo is a flaky failure
Modified: trunk/Tools/Scripts/libraries/webkitscmpy/setup.py (277101 => 277102)
--- trunk/Tools/Scripts/libraries/webkitscmpy/setup.py 2021-05-06 17:20:55 UTC (rev 277101)
+++ trunk/Tools/Scripts/libraries/webkitscmpy/setup.py 2021-05-06 18:32:40 UTC (rev 277102)
@@ -29,7 +29,7 @@
setup(
name='webkitscmpy',
- version='0.13.9',
+ version='0.14.0',
description='Library designed to interact with git and svn repositories.',
long_description=readme(),
classifiers=[
Modified: trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/__init__.py (277101 => 277102)
--- trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/__init__.py 2021-05-06 17:20:55 UTC (rev 277101)
+++ trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/__init__.py 2021-05-06 18:32:40 UTC (rev 277102)
@@ -46,7 +46,7 @@
"Please install webkitcorepy with `pip install webkitcorepy --extra-index-url <package index URL>`"
)
-version = Version(0, 13, 9)
+version = Version(0, 14, 0)
AutoInstall.register(Package('fasteners', Version(0, 15, 0)))
AutoInstall.register(Package('monotonic', Version(1, 5)))
Modified: trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/contributor.py (277101 => 277102)
--- trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/contributor.py 2021-05-06 17:20:55 UTC (rev 277101)
+++ trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/contributor.py 2021-05-06 18:32:40 UTC (rev 277102)
@@ -32,7 +32,8 @@
AUTOMATED_CHECKIN_RE = re.compile(r'Author: (?P<author>.*) <devnull>')
UNKNOWN_AUTHOR = re.compile(r'Author: (?P<author>.*) <None>')
EMPTY_AUTHOR = re.compile(r'Author: (?P<author>.*) <>')
- SVN_AUTHOR_RE = re.compile(r'r\d+ \| (?P<email>.*) \| (?P<date>.*) \| \d+ lines?')
+ SVN_AUTHOR_RE = re.compile(r'r(?P<revision>\d+) \| (?P<email>.*) \| (?P<date>.*) \| \d+ lines?')
+ SVN_AUTHOR_Q_RE = re.compile(r'r(?P<revision>\d+) \| (?P<email>.*) \| (?P<date>.*)')
SVN_PATCH_FROM_RE = re.compile(r'Patch by (?P<author>.*) <(?P<email>.*)> on \d+-\d+-\d+')
class Encoder(json.JSONEncoder):
@@ -115,11 +116,19 @@
email = None
author = None
- for _expression_ in [cls.GIT_AUTHOR_RE, cls.SVN_AUTHOR_RE, cls.SVN_PATCH_FROM_RE, cls.AUTOMATED_CHECKIN_RE, cls.UNKNOWN_AUTHOR, cls.EMPTY_AUTHOR]:
+ for _expression_ in [
+ cls.GIT_AUTHOR_RE,
+ cls.SVN_AUTHOR_RE,
+ cls.SVN_PATCH_FROM_RE,
+ cls.AUTOMATED_CHECKIN_RE,
+ cls.UNKNOWN_AUTHOR,
+ cls.EMPTY_AUTHOR,
+ cls.SVN_AUTHOR_Q_RE,
+ ]:
match = _expression_.match(line)
if match:
if 'author' in _expression_.groupindex:
- author = match.group('author')
+ author = match.group('author').lstrip()
if '(no author)' in author or 'Automated Checkin' in author or 'Unknown' in author:
author = None
if 'email' in _expression_.groupindex:
Modified: trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/local/git.py (277101 => 277102)
--- trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/local/git.py 2021-05-06 17:20:55 UTC (rev 277101)
+++ trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/local/git.py 2021-05-06 18:32:40 UTC (rev 277102)
@@ -20,12 +20,18 @@
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import calendar
import logging
import os
import re
import six
+import subprocess
+import sys
+import time
-from webkitcorepy import run, decorators, TimeoutExpired
+from datetime import datetime, timedelta
+
+from webkitcorepy import run, decorators
from webkitscmpy.local import Scm
from webkitscmpy import Commit, Contributor, log
@@ -300,6 +306,106 @@
message=logcontent if include_log else None,
)
+ def _args_from_content(self, content, include_log=True):
+ author = None
+ timestamp = None
+
+ for line in content.splitlines()[:4]:
+ split = line.split(': ')
+ if split[0] == 'Author':
+ author = Contributor.from_scm_log(line.lstrip(), self.contributors)
+ elif split[0] == 'CommitDate':
+ tz_diff = line.split(' ')[-1]
+ date = datetime.strptime(split[1].lstrip()[:-len(tz_diff)], '%a %b %d %H:%M:%S %Y ')
+ date += timedelta(
+ hours=int(tz_diff[1:3]),
+ minutes=int(tz_diff[3:5]),
+ ) * (1 if tz_diff[0] == '-' else -1)
+ timestamp = int(calendar.timegm(date.timetuple())) - time.timezone
+
+ message = ''
+ for line in content.splitlines()[5:]:
+ message += line[4:] + '\n'
+ matches = self.GIT_SVN_REVISION.findall(message)
+
+ return dict(
+ revision=int(matches[-1].split('@')[0]) if matches else None,
+ author=author,
+ timestamp=timestamp,
+ message=message.rstrip() if include_log else None,
+ )
+
+ def commits(self, begin=None, end=None, include_log=True, include_identifier=True):
+ begin, end = self._commit_range(begin=begin, end=end, include_identifier=include_identifier)
+
+ try:
+ log = None
+ log = subprocess.Popen(
+ [self.executable(), 'log', '--format=fuller', '{}...{}'.format(end.hash, begin.hash)],
+ cwd=self.root_path,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ **(dict(encoding='utf-8') if sys.version_info > (3, 0) else dict())
+ )
+ if log.poll():
+ raise self.Exception("Failed to construct history for '{}'".format(end.branch))
+
+ line = log.stdout.readline()
+ previous = [end]
+ while line:
+ if not line.startswith('commit '):
+ raise OSError('Failed to parse `git log` format')
+ branch_point = previous[0].branch_point
+ identifier = previous[0].identifier
+ hash = line.split(' ')[-1].rstrip()
+ if hash != previous[0].hash:
+ identifier -= 1
+
+ if not identifier:
+ identifier = branch_point
+ branch_point = None
+
+ content = ''
+ line = log.stdout.readline()
+ while line and not line.startswith('commit '):
+ content += line
+ line = log.stdout.readline()
+
+ commit = Commit(
+ repository_id=self.id,
+ hash=hash,
+ branch=end.branch if identifier and branch_point else self.default_branch,
+ identifier=identifier if include_identifier else None,
+ branch_point=branch_point if include_identifier else None,
+ order=0,
+ **self._args_from_content(content, include_log=include_log)
+ )
+
+ # Ensure that we don't duplicate the first and last commits
+ if commit.hash == previous[0].hash:
+ previous[0] = commit
+
+ # If we share a timestamp with the previous commit, that means that this commit has an order
+ # less than the set of commits cached in previous
+ elif commit.timestamp == previous[0].timestamp:
+ for cached in previous:
+ cached.order += 1
+ previous.append(commit)
+
+ # If we don't share a timestamp with the previous set of commits, we should return all commits
+ # cached in previous.
+ else:
+ for cached in previous:
+ yield cached
+ previous = [commit]
+
+ for cached in previous:
+ cached.order += begin.order
+ yield cached
+ finally:
+ if log:
+ log.kill()
+
def find(self, argument, include_log=True, include_identifier=True):
if not isinstance(argument, six.string_types):
raise ValueError("Expected 'argument' to be a string, not '{}'".format(type(argument)))
Modified: trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/local/svn.py (277101 => 277102)
--- trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/local/svn.py 2021-05-06 17:20:55 UTC (rev 277101)
+++ trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/local/svn.py 2021-05-06 18:32:40 UTC (rev 277102)
@@ -401,6 +401,91 @@
message=message,
)
+ def _args_from_content(self, content, include_log=True):
+ leading = content.splitlines()[0]
+ match = Contributor.SVN_AUTHOR_RE.match(leading) or Contributor.SVN_AUTHOR_Q_RE.match(leading)
+ if not match:
+ return {}
+
+ tz_diff = match.group('date').split(' ', 2)[-1]
+ date = datetime.strptime(match.group('date')[:-len(tz_diff)], '%Y-%m-%d %H:%M:%S ')
+ date += timedelta(
+ hours=int(tz_diff[1:3]),
+ minutes=int(tz_diff[3:5]),
+ ) * (1 if tz_diff[0] == '-' else -1)
+
+ return dict(
+ revision=int(match.group('revision')),
+ timestamp=int(calendar.timegm(date.timetuple())),
+ author=Contributor.from_scm_log(leading, self.contributors),
+ message='\n'.join(content.splitlines()[2:]).rstrip() if include_log else None,
+ )
+
+
+ def commits(self, begin=None, end=None, include_log=True, include_identifier=True):
+ begin, end = self._commit_range(begin=begin, end=end, include_identifier=include_identifier)
+ previous = end
+ if end.branch == self.default_branch or '/' in end.branch:
+ branch_arg = '^/{}'.format(end.branch)
+ else:
+ branch_arg = '^/branches/{}'.format(end.branch)
+
+ try:
+ log = None
+ log = subprocess.Popen(
+ [self.executable(), 'log', '-r', '{}:{}'.format(
+ end.revision, begin.revision,
+ ), branch_arg] + ([] if include_log else ['-q']),
+ cwd=self.root_path,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ **(dict(encoding='utf-8') if sys.version_info > (3, 0) else dict())
+ )
+ if log.poll():
+ raise self.Exception('Failed to find commits between {} and {} on {}'.format(begin, end, branch_arg))
+
+ content = ''
+ line = log.stdout.readline()
+ divider = '-' * 72
+ while True:
+ if line and line.rstrip() != divider:
+ content += line
+ line = log.stdout.readline()
+ continue
+
+ if not content:
+ line = log.stdout.readline()
+ continue
+
+ branch_point = previous.branch_point if include_identifier else None
+ identifier = previous.identifier if include_identifier else None
+
+ args = self._args_from_content(content, include_log=include_log)
+ if args['revision'] != previous.revision:
+ yield previous
+ identifier -= 1
+ if not identifier:
+ identifier = branch_point
+ branch_point = None
+
+ previous = Commit(
+ repository_id=self.id,
+ branch=end.branch if branch_point else self.default_branch,
+ identifier=identifier,
+ branch_point=branch_point,
+ **args
+ )
+ content = ''
+ if not line:
+ break
+ line = log.stdout.readline()
+
+ yield previous
+
+ finally:
+ if log:
+ log.kill()
+
def checkout(self, argument):
commit = self.find(argument)
if not commit:
Modified: trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/mocks/local/git.py (277101 => 277102)
--- trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/mocks/local/git.py 2021-05-06 17:20:55 UTC (rev 277101)
+++ trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/mocks/local/git.py 2021-05-06 18:32:40 UTC (rev 277102)
@@ -23,6 +23,7 @@
import json
import os
import re
+import time
from datetime import datetime
from webkitcorepy import mocks, OutputCapture, StringIO
@@ -228,7 +229,7 @@
branch=self.branch,
author=self.find(args[2]).author.name,
email=self.find(args[2]).author.email,
- date=datetime.fromtimestamp(self.find(args[2]).timestamp).strftime('%a %b %d %H:%M:%S %Y'),
+ date=datetime.utcfromtimestamp(self.find(args[2]).timestamp + time.timezone).strftime('%a %b %d %H:%M:%S %Y +0000'),
log='\n'.join([
(' ' + line) if line else '' for line in self.find(args[2]).message.splitlines()
] + ([' git-svn-id: https://svn.{}/repository/{}/trunk@{} 268f45cc-cd09-0410-ab3c-d52691b4dbfc'.format(
@@ -240,6 +241,33 @@
),
) if self.find(args[2]) else mocks.ProcessCompletion(returncode=128),
), mocks.Subprocess.Route(
+ self.executable, 'log', '--format=fuller', re.compile(r'.+\.\.\..+'),
+ cwd=self.path,
+ generator=lambda *args, **kwargs: mocks.ProcessCompletion(
+ returncode=0,
+ stdout='\n'.join([
+ 'commit {hash}\n'
+ 'Author: {author} <{email}>\n'
+ 'AuthorDate: {date}\n'
+ 'Commit: {author} <{email}>\n'
+ 'CommitDate: {date}\n'
+ '\n{log}'.format(
+ hash=commit.hash,
+ author=commit.author.name,
+ email=commit.author.email,
+ date=datetime.utcfromtimestamp(commit.timestamp + time.timezone).strftime('%a %b %d %H:%M:%S %Y +0000'),
+ log='\n'.join([
+ (' ' + line) if line else '' for line in commit.message.splitlines()
+ ] + ([
+ ' git-svn-id: https://svn.{}/repository/{}/trunk@{} 268f45cc-cd09-0410-ab3c-d52691b4dbfc'.format(
+ self.remote.split('@')[-1].split(':')[0],
+ os.path.basename(path),
+ commit.revision,
+ )] if git_svn else []),
+ )) for commit in self.commits_in_range(args[3].split('...')[-1], args[3].split('...')[0])
+ ])
+ )
+ ), mocks.Subprocess.Route(
self.executable, 'rev-list', '--count', '--no-merges', re.compile(r'.+'),
cwd=self.path,
generator=lambda *args, **kwargs: mocks.ProcessCompletion(
@@ -463,3 +491,34 @@
returncode=0,
stdout=stdout.getvalue(),
)
+
+ def commits_in_range(self, begin, end):
+ branches = [self.default_branch]
+ for branch, commits in self.commits.items():
+ if branch == self.default_branch:
+ continue
+ for commit in commits:
+ if commit.hash == end:
+ branches.insert(0, branch)
+ break
+ if len(branches) > 1:
+ break
+
+ in_range = False
+ previous = None
+ for branch in branches:
+ for commit in reversed(self.commits[branch]):
+ if commit.hash == end:
+ in_range = True
+ if in_range and (not previous or commit.hash != previous.hash):
+ yield commit
+ previous = commit
+ if commit.hash == begin:
+ in_range = False
+ in_range = False
+ if not previous or branch == self.default_branch:
+ continue
+
+ for commit in reversed(self.commits[self.default_branch]):
+ if previous.branch_point == commit.identifier:
+ end = commit.hash
Modified: trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/mocks/local/svn.py (277101 => 277102)
--- trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/mocks/local/svn.py 2021-05-06 17:20:55 UTC (rev 277101)
+++ trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/mocks/local/svn.py 2021-05-06 18:32:40 UTC (rev 277102)
@@ -65,14 +65,18 @@
super(Svn, self).__init__(
mocks.Subprocess.Route(
+ self.executable, 'info', self.BRANCH_RE,
+ cwd=self.path,
+ generator=lambda *args, **kwargs: self._info(branch=self.BRANCH_RE.match(args[2]).group('branch'), cwd=kwargs.get('cwd', ''))
+ ), mocks.Subprocess.Route(
+ self.executable, 'info', '-r', re.compile(r'\d+'),
+ cwd=self.path,
+ generator=lambda *args, **kwargs: self._info(revision=int(args[3]), cwd=kwargs.get('cwd', ''))
+ ), mocks.Subprocess.Route(
self.executable, 'info',
cwd=self.path,
generator=lambda *args, **kwargs: self._info(cwd=kwargs.get('cwd', ''))
), mocks.Subprocess.Route(
- self.executable, 'info', self.BRANCH_RE,
- cwd=self.path,
- generator=lambda *args, **kwargs: self._info(branch=self.BRANCH_RE.match(args[2]).group('branch'), cwd=kwargs.get('cwd', ''))
- ), mocks.Subprocess.Route(
self.executable, 'list', '^/branches',
cwd=self.path,
generator=lambda *args, **kwargs: mocks.ProcessCompletion(
@@ -119,6 +123,14 @@
revision=args[5],
) if self.connected else mocks.ProcessCompletion(returncode=1)
), mocks.Subprocess.Route(
+ self.executable, 'log', '-r', re.compile(r'\d+:\d+'), self.BRANCH_RE,
+ cwd=self.path,
+ generator=lambda *args, **kwargs: self._log_range(
+ branch=self.BRANCH_RE.match(args[4]).group('branch'),
+ end=int(args[3].split(':')[0]),
+ begin=int(args[3].split(':')[-1]),
+ ) if self.connected else mocks.ProcessCompletion(returncode=1)
+ ), mocks.Subprocess.Route(
self.executable, 'up', '-r', re.compile(r'\d+'),
cwd=self.path,
generator=lambda *args, **kwargs:
@@ -204,6 +216,28 @@
),
)
+ def _log_range(self, branch=None, end=None, begin=None):
+ if end < begin:
+ return mocks.ProcessCompletion(returncode=1)
+
+ output = ''
+ previous = None
+ for b in [branch, 'trunk']:
+ for candidate in reversed(self.commits.get(b, [])):
+ if candidate.revision > end or candidate.revision < begin:
+ continue
+ if previous and previous.revision <= candidate.revision:
+ continue
+ previous = candidate
+ output += ('------------------------------------------------------------------------\n'
+ '{line} | {lines} lines\n\n'
+ '{log}\n').format(
+ line=self.log_line(candidate),
+ lines=len(candidate.message.splitlines()),
+ log=candidate.message
+ )
+ return mocks.ProcessCompletion(returncode=0, stdout=output)
+
def find(self, branch=None, revision=None):
if not branch and not revision:
return self.head
Modified: trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/mocks/remote/git_hub.py (277101 => 277102)
--- trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/mocks/remote/git_hub.py 2021-05-06 17:20:55 UTC (rev 277101)
+++ trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/mocks/remote/git_hub.py 2021-05-06 18:32:40 UTC (rev 277102)
@@ -132,6 +132,52 @@
} for reference, commit in (self.commits if type == 'branches' else self.tags).items()
], url=""
+ def _commits_response(self, url, ref):
+ from datetime import datetime, timedelta
+
+ base = self.commit(ref)
+ if not base:
+ return mocks.Response(
+ status_code=404,
+ url=""
+ text=json.dumps(dict(message='No commit found for SHA: {}'.format(ref))),
+ )
+
+ response = []
+ for branch in [self.default_branch] if base.branch == self.default_branch else [base.branch, self.default_branch]:
+ in_range = False
+ previous = None
+ for commit in reversed(self.commits[branch]):
+ if commit.hash == ref:
+ in_range = True
+ if not in_range:
+ continue
+ previous = commit
+ response.append({
+ 'sha': commit.hash,
+ 'commit': {
+ 'author': {
+ 'name': commit.author.name,
+ 'email': commit.author.email,
+ 'date': datetime.utcfromtimestamp(commit.timestamp - timedelta(hours=7).seconds).strftime('%Y-%m-%dT%H:%M:%SZ'),
+ }, 'committer': {
+ 'name': commit.author.name,
+ 'email': commit.author.email,
+ 'date': datetime.utcfromtimestamp(commit.timestamp - timedelta(hours=7).seconds).strftime('%Y-%m-%dT%H:%M:%SZ'),
+ }, 'message': commit.message + ('\ngit-svn-id: https://svn.example.org/repository/webkit/{}@{} 268f45cc-cd09-0410-ab3c-d52691b4dbfc\n'.format(
+ 'trunk' if commit.branch == self.default_branch else commit.branch, commit.revision,
+ ) if commit.revision else ''),
+ 'url': 'https://{}/git/commits/{}'.format(self.api_remote, commit.hash),
+ }, 'url': 'https://{}/commits/{}'.format(self.api_remote, commit.hash),
+ 'html_url': 'https://{}/commit/{}'.format(self.remote, commit.hash),
+ })
+ if branch != self.default_branch:
+ for commit in reversed(self.commits[self.default_branch]):
+ if previous.branch_point == commit.identifier:
+ ref = commit.hash
+
+ return mocks.Response.fromJson(response, url=""
+
def _commit_response(self, url, ref):
from datetime import datetime, timedelta
@@ -153,9 +199,9 @@
'name': commit.author.name,
'email': commit.author.email,
'date': datetime.utcfromtimestamp(commit.timestamp - timedelta(hours=7).seconds).strftime('%Y-%m-%dT%H:%M:%SZ'),
- }, 'message': commit.message + '\ngit-svn-id: https://svn.example.org/repository/webkit/{}@{} 268f45cc-cd09-0410-ab3c-d52691b4dbfc\n'.format(
+ }, 'message': commit.message + ('\ngit-svn-id: https://svn.example.org/repository/webkit/{}@{} 268f45cc-cd09-0410-ab3c-d52691b4dbfc\n'.format(
'trunk' if commit.branch == self.default_branch else commit.branch, commit.revision,
- ) if commit.revision else '',
+ ) if commit.revision else ''),
'url': 'https://{}/git/commits/{}'.format(self.api_remote, commit.hash),
}, 'url': 'https://{}/commits/{}'.format(self.api_remote, commit.hash),
'html_url': 'https://{}/commit/{}'.format(self.remote, commit.hash),
@@ -233,10 +279,11 @@
), url=""
)
- def request(self, method, url, data="" **kwargs):
+ def request(self, method, url, data="" params=None, **kwargs):
if not url.startswith('http://') and not url.startswith('https://'):
return mocks.Response.create404(url)
+ params = params or {}
stripped_url = url.split('://')[-1]
# Top-level API request
@@ -247,6 +294,10 @@
if stripped_url in ['{}/branches'.format(self.api_remote), '{}/tags'.format(self.api_remote)]:
return self._list_refs_response(url="" type=stripped_url.split('/')[-1])
+ # Return a commit and it's parents
+ if stripped_url == '{}/commits'.format(self.api_remote) and params.get('sha'):
+ return self._commits_response(url="" ref=params['sha'])
+
# Extract single commit
if stripped_url.startswith('{}/commits/'.format(self.api_remote)):
return self._commit_response(url="" ref=stripped_url.split('/')[-1])
Modified: trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/mocks/remote/svn.py (277101 => 277102)
--- trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/mocks/remote/svn.py 2021-05-06 17:20:55 UTC (rev 277101)
+++ trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/mocks/remote/svn.py 2021-05-06 18:32:40 UTC (rev 277102)
@@ -106,31 +106,18 @@
if category and category.startswith('branches/'):
category = category.split('/')[-1]
+ category = [category] if category else self.branches(start) + self.tags(start)
- if not category:
- for commits in self.commits.values():
- for commit in commits:
- if commit.revision == start:
- category = commit.branch
- break
+ previous = None
+ for b in category + ['trunk']:
+ for candidate in reversed(self.commits.get(b, [])):
+ if candidate.revision > start or candidate.revision < end:
+ continue
+ if previous and previous.revision <= candidate.revision:
+ continue
+ previous = candidate
+ yield candidate
- if not category:
- return []
-
- result = [commit for commit in reversed(self.commits[category])]
- if self.commits[category][0].branch_point:
- result += [commit for commit in reversed(self.commits['trunk'][:self.commits[category][0].branch_point])]
-
- for index in reversed(range(len(result))):
- if result[index].revision < end:
- result = result[:index]
- continue
- if result[index].revision > start:
- result = result[index:]
- break
-
- return result
-
def request(self, method, url, data="" **kwargs):
from datetime import datetime, timedelta
@@ -266,11 +253,11 @@
# Log for commit
if method == 'REPORT' and stripped_url.startswith('{}!'.format(self.remote)) and match and data.get('S:log-report'):
- commits = self.range(
+ commits = list(self.range(
category=match.group('category'),
start=int(data['S:log-report']['S:start-revision']),
end=int(data['S:log-report']['S:end-revision']),
- )
+ ))
limit = int(data['S:log-report'].get('S:limit', 0))
if limit and len(commits) > limit:
Modified: trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/remote/git_hub.py (277101 => 277102)
--- trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/remote/git_hub.py 2021-05-06 17:20:55 UTC (rev 277101)
+++ trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/remote/git_hub.py 2021-05-06 18:32:40 UTC (rev 277102)
@@ -75,7 +75,7 @@
def is_git(self):
return True
- def request(self, path=None, params=None, headers=None, authenticated=None):
+ def request(self, path=None, params=None, headers=None, authenticated=None, paginate=True):
headers = {key: value for key, value in headers.items()} if headers else dict()
headers['Accept'] = headers.get('Accept', 'application/vnd.github.v3+json')
@@ -105,7 +105,7 @@
return None
result = response.json()
- while isinstance(response.json(), list) and len(response.json()) == params['per_page']:
+ while paginate and isinstance(response.json(), list) and len(response.json()) == params['per_page']:
params['page'] += 1
response = requests.get(url, params=params, headers=headers, auth=auth)
if response.status_code != 200:
@@ -287,16 +287,22 @@
# it's possible for a series of commits to share a commit time. To handle this case, we assign each commit a
# zero-indexed "order" within it's timestamp.
order = 0
- while not identifier or order + 1 < identifier + (branch_point or 0):
- response = self.request('commits/{}'.format('{}~{}'.format(commit_data['sha'], order + 1)))
- if not response:
+ lhash = commit_data['sha']
+ while lhash:
+ response = self.request('commits', paginate=False, params=dict(sha=lhash, per_page=20))
+ if len(response) <= 1:
break
- parent_timestamp = int(calendar.timegm(datetime.strptime(
- response['commit']['committer']['date'], '%Y-%m-%dT%H:%M:%SZ',
- ).timetuple()))
- if parent_timestamp != timestamp:
- break
- order += 1
+ for c in response:
+ if lhash == c['sha']:
+ continue
+ parent_timestamp = int(calendar.timegm(datetime.strptime(
+ c['commit']['committer']['date'], '%Y-%m-%dT%H:%M:%SZ',
+ ).timetuple()))
+ if parent_timestamp != timestamp:
+ lhash = None
+ break
+ lhash = c['sha']
+ order += 1
return Commit(
repository_id=self.id,
@@ -313,6 +319,67 @@
), message=commit_data['commit']['message'] if include_log else None,
)
+ def commits(self, begin=None, end=None, include_log=True, include_identifier=True):
+ begin, end = self._commit_range(begin=begin, end=end, include_identifier=include_identifier)
+
+ previous = end
+ cached = [previous]
+ while previous:
+ response = self.request('commits', paginate=False, params=dict(sha=previous.hash))
+ if not response:
+ break
+ for commit_data in response:
+ branch_point = previous.branch_point
+ identifier = previous.identifier
+ if commit_data['sha'] == previous.hash:
+ cached = cached[:-1]
+ else:
+ identifier -= 1
+
+ if not identifier:
+ identifier = branch_point
+ branch_point = None
+
+ matches = self.GIT_SVN_REVISION.findall(commit_data['commit']['message'])
+ revision = int(matches[-1].split('@')[0]) if matches else None
+
+ email_match = self.EMAIL_RE.match(commit_data['commit']['author']['email'])
+ timestamp = int(calendar.timegm(datetime.strptime(
+ commit_data['commit']['committer']['date'], '%Y-%m-%dT%H:%M:%SZ',
+ ).timetuple()))
+
+ previous = Commit(
+ repository_id=self.id,
+ hash=commit_data['sha'],
+ revision=revision,
+ branch=end.branch if identifier and branch_point else self.default_branch,
+ identifier=identifier if include_identifier else None,
+ branch_point=branch_point if include_identifier else None,
+ timestamp=timestamp,
+ author=self.contributors.create(
+ commit_data['commit']['author']['name'],
+ email_match.group('email') if email_match else None,
+ ), order=0,
+ message=commit_data['commit']['message'] if include_log else None,
+ )
+ if not cached or cached[0].timestamp != previous.timestamp:
+ for c in cached:
+ yield c
+ cached = [previous]
+ else:
+ for c in cached:
+ c.order += 1
+ cached.append(previous)
+
+ if previous.hash == begin.hash or previous.timestamp < begin.timestamp:
+ previous = None
+ break
+
+ for c in cached:
+ c.order += begin.order
+ yield c
+
+
def find(self, argument, include_log=True, include_identifier=True):
if not isinstance(argument, six.string_types):
raise ValueError("Expected 'argument' to be a string, not '{}'".format(type(argument)))
Modified: trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/remote/svn.py (277101 => 277102)
--- trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/remote/svn.py 2021-05-06 17:20:55 UTC (rev 277101)
+++ trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/remote/svn.py 2021-05-06 18:32:40 UTC (rev 277102)
@@ -32,14 +32,14 @@
from datetime import datetime
-from webkitcorepy import log, run, decorators
+from webkitcorepy import decorators, string_utils
from webkitscmpy.remote.scm import Scm
-from webkitscmpy import Commit, Contributor, Version
+from webkitscmpy import Commit, Version
class Svn(Scm):
URL_RE = re.compile(r'\Ahttps?://svn.(?P<host>\S+)/repository/\S+\Z')
- HISTORY_RE = re.compile(b'<D:version-name>(?P<revision>\d+)</D:version-name>')
+ DATA_RE = re.compile(b'<[SD]:(?P<tag>\S+)>(?P<content>.*)</[SD]:.+>')
CACHE_VERSION = Version(1)
@classmethod
@@ -244,8 +244,8 @@
default_count = 0
for line in response.iter_lines():
- match = self.HISTORY_RE.match(line)
- if not match:
+ match = self.DATA_RE.match(line)
+ if not match or match.group('tag') != b'version-name':
continue
if not did_warn:
@@ -254,7 +254,7 @@
self.log('Caching commit data for {}, this will take a few minutes...'.format(branch))
did_warn = True
- revision = int(match.group('revision'))
+ revision = int(match.group('content'))
if pos > 0 and self._metadata_cache[branch][pos - 1] == revision:
break
if not is_default_branch:
@@ -455,3 +455,67 @@
author=author,
message=message,
)
+
+ def _args_from_content(self, content, include_log=True):
+ xml = xmltodict.parse(content)
+ date = datetime.strptime(string_utils.decode(xml['S:log-item']['S:date']).split('.')[0], '%Y-%m-%dT%H:%M:%S')
+ name = string_utils.decode(xml['S:log-item']['D:creator-displayname'])
+
+ return dict(
+ revision=int(xml['S:log-item']['D:version-name']),
+ author=self.contributors.create(name, name) if name and '@' in name else self.contributors.create(name),
+ timestamp=int(calendar.timegm(date.timetuple())),
+ message=string_utils.decode(xml['S:log-item']['D:comment']) if include_log else None,
+ )
+
+ def commits(self, begin=None, end=None, include_log=True, include_identifier=True):
+ begin, end = self._commit_range(begin=begin, end=end, include_identifier=include_identifier)
+ previous = end
+
+ content = b''
+ with requests.request(
+ method='REPORT',
+ url=''.format(
+ self.url,
+ end.revision,
+ end.branch if end.branch == self.default_branch or '/' in end.branch else 'branches/{}'.format(end.branch),
+ ), stream=True,
+ headers={
+ 'Content-Type': 'text/xml',
+ 'Accept-Encoding': 'gzip',
+ 'DEPTH': '1',
+ }, data=''
+ '<S:start-revision>{end}</S:start-revision>\n'
+ '<S:end-revision>{begin}</S:end-revision>\n'
+ '<S:path></S:path>\n'
+ '</S:log-report>\n'.format(end=end.revision, begin=begin.revision),
+ ) as response:
+ if response.status_code != 200:
+ raise self.Exception("Failed to construct branch history for '{}'".format(branch))
+ for line in response.iter_lines():
+ if line == b'<S:log-item>':
+ content = line + b'\n'
+ else:
+ content += line + b'\n'
+ if line != b'</S:log-item>':
+ continue
+
+ args = self._args_from_content(content, include_log=include_log)
+
+ branch_point = previous.branch_point if include_identifier else None
+ identifier = previous.identifier if include_identifier else None
+ if args['revision'] != previous.revision:
+ identifier -= 1
+ if not identifier:
+ identifier = branch_point
+ branch_point = None
+
+ previous = Commit(
+ repository_id=self.id,
+ branch=end.branch if branch_point else self.default_branch,
+ identifier=identifier,
+ branch_point=branch_point,
+ **args
+ )
+ yield previous
+ content = b''
Modified: trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/scm_base.py (277101 => 277102)
--- trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/scm_base.py 2021-05-06 17:20:55 UTC (rev 277101)
+++ trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/scm_base.py 2021-05-06 18:32:40 UTC (rev 277102)
@@ -73,6 +73,38 @@
def commit(self, hash=None, revision=None, identifier=None, branch=None, tag=None, include_log=True, include_identifier=True):
raise NotImplementedError()
+ def _commit_range(self, begin=None, end=None, include_log=False, include_identifier=True):
+ begin_args = begin or dict()
+ end_args = end or dict()
+
+ if not begin_args:
+ raise TypeError("_commit_range() missing required 'begin' arguments")
+ if not end_args:
+ raise TypeError("_commit_range() missing required 'end' arguments")
+
+ if list(begin_args.keys()) == ['argument']:
+ begin_result = self.find(include_log=include_log, include_identifier=False, **begin_args)
+ else:
+ begin_result = self.commit(include_log=include_log, include_identifier=False, **begin_args)
+
+ if list(end_args.keys()) == ['argument']:
+ end_result = self.find(include_log=include_log, include_identifier=include_identifier, **end_args)
+ else:
+ end_result = self.commit(include_log=include_log, include_identifier=include_identifier, **end_args)
+
+ if not begin_result:
+ raise TypeError("'{}' failed to define begin in _commit_range()".format(begin_args))
+ if not end_result:
+ raise TypeError("'{}' failed to define begin in _commit_range()".format(end_args))
+ if begin_result.timestamp > end_result.timestamp:
+ raise TypeError("'{}' pre-dates '{}' in _commit_range()".format(begin_result, end_result))
+ if end_result.branch == self.default_branch and begin_result.branch != self.default_branch:
+ raise TypeError("'{}' and '{}' do not share linear history".format(begin_result, end_result))
+ return begin_result, end_result
+
+ def commits(self, begin=None, end=None, include_log=True, include_identifier=True):
+ raise NotImplementedError()
+
def prioritize_branches(self, branches):
if len(branches) == 1:
return branches[0]
Modified: trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/test/find_unittest.py (277101 => 277102)
--- trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/test/find_unittest.py 2021-05-06 17:20:55 UTC (rev 277101)
+++ trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/test/find_unittest.py 2021-05-06 18:32:40 UTC (rev 277102)
@@ -231,7 +231,7 @@
Date: {}
Revision: 4
Identifier: 3@trunk
-'''.format(datetime.fromtimestamp(1601686700).strftime('%a %b %d %H:%M:%S %Y')),
+'''.format(datetime.fromtimestamp(1601684700).strftime('%a %b %d %H:%M:%S %Y')),
)
Modified: trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/test/git_unittest.py (277101 => 277102)
--- trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/test/git_unittest.py 2021-05-06 17:20:55 UTC (rev 277101)
+++ trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/test/git_unittest.py 2021-05-06 18:32:40 UTC (rev 277102)
@@ -28,7 +28,7 @@
from datetime import datetime
from webkitcorepy import LoggerCapture, OutputCapture
from webkitcorepy.mocks import Time as MockTime
-from webkitscmpy import local, mocks, remote
+from webkitscmpy import Commit, local, mocks, remote
class TestGit(unittest.TestCase):
@@ -283,7 +283,29 @@
self.assertEqual(0, local.Git(self.path).commit(hash='bae5d1e90999').order)
self.assertEqual(1, local.Git(self.path).commit(hash='d8bce26fa65c').order)
+ def test_commits(self):
+ for mock in [mocks.local.Git(self.path), mocks.local.Git(self.path, git_svn=True)]:
+ with mock:
+ git = local.Git(self.path)
+ self.assertEqual(Commit.Encoder().default([
+ git.commit(hash='bae5d1e9'),
+ git.commit(hash='1abe25b4'),
+ git.commit(hash='fff83bb2'),
+ git.commit(hash='9b8311f2'),
+ ]), Commit.Encoder().default(list(git.commits(begin=dict(hash='9b8311f2'), end=dict(hash='bae5d1e9')))))
+ def test_commits_branch(self):
+ for mock in [mocks.local.Git(self.path), mocks.local.Git(self.path, git_svn=True)]:
+ with mock:
+ git = local.Git(self.path)
+ self.assertEqual(Commit.Encoder().default([
+ git.commit(hash='621652ad'),
+ git.commit(hash='a30ce849'),
+ git.commit(hash='fff83bb2'),
+ git.commit(hash='9b8311f2'),
+ ]), Commit.Encoder().default(list(git.commits(begin=dict(argument='9b8311f2'), end=dict(argument='621652ad')))))
+
+
class TestGitHub(unittest.TestCase):
remote = 'https://github.example.com/WebKit/WebKit'
@@ -415,7 +437,28 @@
def test_id(self):
self.assertEqual(remote.GitHub(self.remote).id, 'webkit')
+ def test_commits(self):
+ with mocks.remote.GitHub():
+ git = remote.GitHub(self.remote)
+ self.assertEqual(Commit.Encoder().default([
+ git.commit(hash='bae5d1e9'),
+ git.commit(hash='1abe25b4'),
+ git.commit(hash='fff83bb2'),
+ git.commit(hash='9b8311f2'),
+ ]), Commit.Encoder().default(list(git.commits(begin=dict(hash='9b8311f2'), end=dict(hash='bae5d1e9')))))
+ def test_commits_branch(self):
+ with mocks.remote.GitHub():
+ git = remote.GitHub(self.remote)
+ self.assertEqual(Commit.Encoder().default([
+ git.commit(hash='621652ad'),
+ git.commit(hash='a30ce849'),
+ git.commit(hash='fff83bb2'),
+ git.commit(hash='9b8311f2'),
+ ]), Commit.Encoder().default(list(git.commits(begin=dict(argument='9b8311f2'), end=dict(argument='621652ad')))))
+
+
+
class TestBitBucket(unittest.TestCase):
remote = 'https://bitbucket.example.com/projects/WEBKIT/repos/webkit'
Modified: trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/test/svn_unittest.py (277101 => 277102)
--- trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/test/svn_unittest.py 2021-05-06 17:20:55 UTC (rev 277101)
+++ trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/test/svn_unittest.py 2021-05-06 18:32:40 UTC (rev 277102)
@@ -27,7 +27,7 @@
from datetime import datetime, timedelta
from webkitcorepy import OutputCapture
-from webkitscmpy import local, mocks, remote
+from webkitscmpy import Commit, local, mocks, remote
class TestLocalSvn(unittest.TestCase):
@@ -231,7 +231,27 @@
with mocks.local.Svn(self.path), OutputCapture():
self.assertIsNone(local.Svn(self.path).find('trunk', include_identifier=False).identifier)
+ def test_commits(self):
+ with mocks.local.Svn(self.path), OutputCapture():
+ svn = local.Svn(self.path)
+ self.assertEqual(Commit.Encoder().default([
+ svn.commit(revision='r6'),
+ svn.commit(revision='r4'),
+ svn.commit(revision='r2'),
+ svn.commit(revision='r1'),
+ ]), Commit.Encoder().default(list(svn.commits(begin=dict(revision='r1'), end=dict(revision='r6')))))
+ def test_commits_branch(self):
+ with mocks.local.Svn(self.path), OutputCapture():
+ svn = local.Svn(self.path)
+ self.assertEqual(Commit.Encoder().default([
+ svn.commit(revision='r7'),
+ svn.commit(revision='r3'),
+ svn.commit(revision='r2'),
+ svn.commit(revision='r1'),
+ ]), Commit.Encoder().default(list(svn.commits(begin=dict(argument='r1'), end=dict(argument='r7')))))
+
+
class TestRemoteSvn(unittest.TestCase):
remote = 'https://svn.example.org/repository/webkit'
@@ -331,3 +351,24 @@
def test_id(self):
self.assertEqual(remote.Svn(self.remote).id, 'webkit')
+
+ def test_commits(self):
+ self.maxDiff = None
+ with mocks.remote.Svn():
+ svn = remote.Svn(self.remote)
+ self.assertEqual(Commit.Encoder().default([
+ svn.commit(revision='r6'),
+ svn.commit(revision='r4'),
+ svn.commit(revision='r2'),
+ svn.commit(revision='r1'),
+ ]), Commit.Encoder().default(list(svn.commits(begin=dict(revision='r1'), end=dict(revision='r6')))))
+
+ def test_commits_branch(self):
+ with mocks.remote.Svn(), OutputCapture():
+ svn = remote.Svn(self.remote)
+ self.assertEqual(Commit.Encoder().default([
+ svn.commit(revision='r7'),
+ svn.commit(revision='r3'),
+ svn.commit(revision='r2'),
+ svn.commit(revision='r1'),
+ ]), Commit.Encoder().default(list(svn.commits(begin=dict(argument='r1'), end=dict(argument='r7')))))
