Closed by commit rHG92593d72e10b: automation: implement "publish-windows-artifacts" command (authored by indygreg). This revision was automatically updated to reflect the committed changes.
REPOSITORY rHG Mercurial CHANGES SINCE LAST UPDATE https://phab.mercurial-scm.org/D6786?vs=16389&id=16458 CHANGES SINCE LAST ACTION https://phab.mercurial-scm.org/D6786/new/ REVISION DETAIL https://phab.mercurial-scm.org/D6786 AFFECTED FILES contrib/automation/README.rst contrib/automation/hgautomation/cli.py contrib/automation/hgautomation/pypi.py contrib/automation/hgautomation/windows.py contrib/automation/requirements.txt contrib/automation/requirements.txt.in tests/test-check-code.t CHANGE DETAILS diff --git a/tests/test-check-code.t b/tests/test-check-code.t --- a/tests/test-check-code.t +++ b/tests/test-check-code.t @@ -16,6 +16,7 @@ Skipping contrib/automation/hgautomation/aws.py it has no-che?k-code (glob) Skipping contrib/automation/hgautomation/cli.py it has no-che?k-code (glob) Skipping contrib/automation/hgautomation/linux.py it has no-che?k-code (glob) + Skipping contrib/automation/hgautomation/pypi.py it has no-che?k-code (glob) Skipping contrib/automation/hgautomation/ssh.py it has no-che?k-code (glob) Skipping contrib/automation/hgautomation/windows.py it has no-che?k-code (glob) Skipping contrib/automation/hgautomation/winrm.py it has no-che?k-code (glob) diff --git a/contrib/automation/requirements.txt.in b/contrib/automation/requirements.txt.in --- a/contrib/automation/requirements.txt.in +++ b/contrib/automation/requirements.txt.in @@ -1,3 +1,4 @@ boto3 paramiko pypsrp +twine diff --git a/contrib/automation/requirements.txt b/contrib/automation/requirements.txt --- a/contrib/automation/requirements.txt +++ b/contrib/automation/requirements.txt @@ -26,6 +26,10 @@ --hash=sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7 \ --hash=sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc \ # via paramiko +bleach==3.1.0 \ + --hash=sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16 \ + --hash=sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa \ + # via readme-renderer boto3==1.9.223 \ --hash=sha256:12ceb047c3cfbd2363b35e1c24b082808a1bb9b90f4f0b7375e83d21015bf47b \ --hash=sha256:6e833a9068309c24d7752e280b2925cf5968a88111bc95fcebc451a09f8b424e @@ -93,7 +97,7 @@ --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \ --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \ --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99 \ - # via botocore + # via botocore, readme-renderer idna==2.8 \ --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \ @@ -109,9 +113,17 @@ paramiko==2.6.0 \ --hash=sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf \ --hash=sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041 +pkginfo==1.5.0.1 \ + --hash=sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb \ + --hash=sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32 \ + # via twine pycparser==2.19 \ --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \ # via cffi +pygments==2.4.2 \ + --hash=sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127 \ + --hash=sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297 \ + # via readme-renderer pynacl==1.3.0 \ --hash=sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255 \ --hash=sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c \ @@ -140,10 +152,18 @@ --hash=sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb \ --hash=sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e \ # via botocore +readme-renderer==24.0 \ + --hash=sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f \ + --hash=sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d \ + # via twine +requests-toolbelt==0.9.1 \ + --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ + --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 \ + # via twine requests==2.22.0 \ --hash=sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4 \ --hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31 \ - # via pypsrp + # via pypsrp, requests-toolbelt, twine s3transfer==0.2.1 \ --hash=sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d \ --hash=sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba \ @@ -151,8 +171,23 @@ six==1.12.0 \ --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \ - # via bcrypt, cryptography, pynacl, pypsrp, python-dateutil + # via bcrypt, bleach, cryptography, pynacl, pypsrp, python-dateutil, readme-renderer +tqdm==4.35.0 \ + --hash=sha256:1be3e4e3198f2d0e47b928e9d9a8ec1b63525db29095cec1467f4c5a4ea8ebf9 \ + --hash=sha256:7e39a30e3d34a7a6539378e39d7490326253b7ee354878a92255656dc4284457 \ + # via twine +twine==1.13.0 \ + --hash=sha256:0fb0bfa3df4f62076cab5def36b1a71a2e4acb4d1fa5c97475b048117b1a6446 \ + --hash=sha256:d6c29c933ecfc74e9b1d9fa13aa1f87c5d5770e119f5a4ce032092f0ff5b14dc urllib3==1.25.3 \ --hash=sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1 \ --hash=sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232 \ # via botocore, requests +webencodings==0.5.1 \ + --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \ + --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 \ + # via bleach + +# WARNING: The following packages were not pinned, but pip requires them to be +# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag. +# setuptools==41.2.0 # via twine diff --git a/contrib/automation/hgautomation/windows.py b/contrib/automation/hgautomation/windows.py --- a/contrib/automation/hgautomation/windows.py +++ b/contrib/automation/hgautomation/windows.py @@ -7,12 +7,17 @@ # no-check-code because Python 3 native. +import datetime import os +import paramiko import pathlib import re import subprocess import tempfile +from .pypi import ( + upload as pypi_upload, +) from .winrm import ( run_powershell, ) @@ -100,6 +105,26 @@ }} ''' +X86_WHEEL_FILENAME = 'mercurial-{version}-cp27-cp27m-win32.whl' +X64_WHEEL_FILENAME = 'mercurial-{version}-cp27-cp27m-win_amd64.whl' +X86_EXE_FILENAME = 'Mercurial-{version}.exe' +X64_EXE_FILENAME = 'Mercurial-{version}-x64.exe' +X86_MSI_FILENAME = 'mercurial-{version}-x86.msi' +X64_MSI_FILENAME = 'mercurial-{version}-x64.msi' + +MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows' + +X86_USER_AGENT_PATTERN = '.*Windows.*' +X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*' + +X86_EXE_DESCRIPTION = ('Mercurial {version} Inno Setup installer - x86 Windows ' + '- does not require admin rights') +X64_EXE_DESCRIPTION = ('Mercurial {version} Inno Setup installer - x64 Windows ' + '- does not require admin rights') +X86_MSI_DESCRIPTION = ('Mercurial {version} MSI installer - x86 Windows ' + '- requires admin rights') +X64_MSI_DESCRIPTION = ('Mercurial {version} MSI installer - x64 Windows ' + '- requires admin rights') def get_vc_prefix(arch): if arch == 'x86': @@ -296,3 +321,152 @@ ) run_powershell(winrm_client, ps) + + +def resolve_wheel_artifacts(dist_path: pathlib.Path, version: str): + return ( + dist_path / X86_WHEEL_FILENAME.format(version=version), + dist_path / X64_WHEEL_FILENAME.format(version=version), + ) + + +def resolve_all_artifacts(dist_path: pathlib.Path, version: str): + return ( + dist_path / X86_WHEEL_FILENAME.format(version=version), + dist_path / X64_WHEEL_FILENAME.format(version=version), + dist_path / X86_EXE_FILENAME.format(version=version), + dist_path / X64_EXE_FILENAME.format(version=version), + dist_path / X86_MSI_FILENAME.format(version=version), + dist_path / X64_MSI_FILENAME.format(version=version), + ) + + +def generate_latest_dat(version: str): + x86_exe_filename = X86_EXE_FILENAME.format(version=version) + x64_exe_filename = X64_EXE_FILENAME.format(version=version) + x86_msi_filename = X86_MSI_FILENAME.format(version=version) + x64_msi_filename = X64_MSI_FILENAME.format(version=version) + + entries = ( + ( + '10', + version, + X86_USER_AGENT_PATTERN, + '%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_exe_filename), + X86_EXE_DESCRIPTION.format(version=version), + ), + ( + '10', + version, + X64_USER_AGENT_PATTERN, + '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_exe_filename), + X64_EXE_DESCRIPTION.format(version=version), + ), + ( + '10', + version, + X86_USER_AGENT_PATTERN, + '%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_msi_filename), + X86_MSI_DESCRIPTION.format(version=version), + ), + ( + '10', + version, + X64_USER_AGENT_PATTERN, + '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_msi_filename), + X64_MSI_DESCRIPTION.format(version=version) + ) + ) + + lines = ['\t'.join(e) for e in entries] + + return '\n'.join(lines) + '\n' + + +def publish_artifacts_pypi(dist_path: pathlib.Path, version: str): + """Publish Windows release artifacts to PyPI.""" + + wheel_paths = resolve_wheel_artifacts(dist_path, version) + + for p in wheel_paths: + if not p.exists(): + raise Exception('%s not found' % p) + + print('uploading wheels to PyPI (you may be prompted for credentials)') + pypi_upload(wheel_paths) + + +def publish_artifacts_mercurial_scm_org(dist_path: pathlib.Path, version: str, + ssh_username=None): + """Publish Windows release artifacts to mercurial-scm.org.""" + all_paths = resolve_all_artifacts(dist_path, version) + + for p in all_paths: + if not p.exists(): + raise Exception('%s not found' % p) + + client = paramiko.SSHClient() + client.load_system_host_keys() + # We assume the system SSH configuration knows how to connect. + print('connecting to mercurial-scm.org via ssh...') + try: + client.connect('mercurial-scm.org', username=ssh_username) + except paramiko.AuthenticationException: + print('error authenticating; is an SSH key available in an SSH agent?') + raise + + print('SSH connection established') + + print('opening SFTP client...') + sftp = client.open_sftp() + print('SFTP client obtained') + + for p in all_paths: + dest_path = '/var/www/release/windows/%s' % p.name + print('uploading %s to %s' % (p, dest_path)) + + with p.open('rb') as fh: + data = fh.read() + + with sftp.open(dest_path, 'wb') as fh: + fh.write(data) + fh.chmod(0o0664) + + latest_dat_path = '/var/www/release/windows/latest.dat' + + now = datetime.datetime.utcnow() + backup_path = dist_path / ( + 'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S')) + print('backing up %s to %s' % (latest_dat_path, backup_path)) + + with sftp.open(latest_dat_path, 'rb') as fh: + latest_dat_old = fh.read() + + with backup_path.open('wb') as fh: + fh.write(latest_dat_old) + + print('writing %s with content:' % latest_dat_path) + latest_dat_content = generate_latest_dat(version) + print(latest_dat_content) + + with sftp.open(latest_dat_path, 'wb') as fh: + fh.write(latest_dat_content.encode('ascii')) + + +def publish_artifacts(dist_path: pathlib.Path, version: str, + pypi=True, mercurial_scm_org=True, + ssh_username=None): + """Publish Windows release artifacts. + + Files are found in `dist_path`. We will look for files with version string + `version`. + + `pypi` controls whether we upload to PyPI. + `mercurial_scm_org` controls whether we upload to mercurial-scm.org. + """ + if pypi: + publish_artifacts_pypi(dist_path, version) + + if mercurial_scm_org: + publish_artifacts_mercurial_scm_org(dist_path, version, + ssh_username=ssh_username) diff --git a/contrib/automation/hgautomation/pypi.py b/contrib/automation/hgautomation/pypi.py new file mode 100644 --- /dev/null +++ b/contrib/automation/hgautomation/pypi.py @@ -0,0 +1,25 @@ +# pypi.py - Automation around PyPI +# +# Copyright 2019 Gregory Szorc <gregory.sz...@gmail.com> +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +# no-check-code because Python 3 native. + +from twine.commands.upload import ( + upload as twine_upload, +) +from twine.settings import ( + Settings, +) + + +def upload(paths): + """Upload files to PyPI. + + `paths` is an iterable of `pathlib.Path`. + """ + settings = Settings() + + twine_upload(settings, [str(p) for p in paths]) diff --git a/contrib/automation/hgautomation/cli.py b/contrib/automation/hgautomation/cli.py --- a/contrib/automation/hgautomation/cli.py +++ b/contrib/automation/hgautomation/cli.py @@ -185,6 +185,14 @@ test_flags) +def publish_windows_artifacts(hg: HGAutomation, aws_region, version: str, + pypi: bool, mercurial_scm_org: bool, + ssh_username: str): + windows.publish_artifacts(DIST_PATH, version, + pypi=pypi, mercurial_scm_org=mercurial_scm_org, + ssh_username=ssh_username) + + def get_parser(): parser = argparse.ArgumentParser() @@ -403,6 +411,34 @@ ) sp.set_defaults(func=run_tests_windows) + sp = subparsers.add_parser( + 'publish-windows-artifacts', + help='Publish built Windows artifacts (wheels, installers, etc)' + ) + sp.add_argument( + '--no-pypi', + dest='pypi', + action='store_false', + default=True, + help='Skip uploading to PyPI', + ) + sp.add_argument( + '--no-mercurial-scm-org', + dest='mercurial_scm_org', + action='store_false', + default=True, + help='Skip uploading to www.mercurial-scm.org', + ) + sp.add_argument( + '--ssh-username', + help='SSH username for mercurial-scm.org', + ) + sp.add_argument( + 'version', + help='Mercurial version string to locate local packages', + ) + sp.set_defaults(func=publish_windows_artifacts) + return parser diff --git a/contrib/automation/README.rst b/contrib/automation/README.rst --- a/contrib/automation/README.rst +++ b/contrib/automation/README.rst @@ -181,3 +181,25 @@ Documenting them is beyond the scope of this document. Various tests also require other optional dependencies and missing dependencies will be printed by the test runner when a test is skipped. + +Releasing Windows Artifacts +=========================== + +The `automation.py` script can be used to automate the release of Windows +artifacts:: + + $ ./automation.py build-all-windows-packages --revision 5.1.1 + $ ./automation.py publish-windows-artifacts 5.1.1 + +The first command will launch an EC2 instance to build all Windows packages +and copy them into the `dist` directory relative to the repository root. The +second command will then attempt to upload these files to PyPI (via `twine`) +and to `mercurial-scm.org` (via SSH). + +Uploading to PyPI requires a PyPI account with write access to the `Mercurial` +package. You can skip PyPI uploading by passing `--no-pypi`. + +Uploading to `mercurial-scm.org` requires an SSH account on that server +with `windows` group membership and for the SSH key for that account to be the +default SSH key (e.g. `~/.ssh/id_rsa`) or in a running SSH agent. You can +skip `mercurial-scm.org` uploading by passing `--no-mercurial-scm-org`. To: indygreg, #hg-reviewers, pulkit Cc: mercurial-devel _______________________________________________ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
