Hi, On Wed, 5 Mar 2025 at 18:51, Andres Freund <and...@anarazel.de> wrote: > > Hi, > > I'm inclined to think we should apply to this to all branches with CI support, > not just master. It's kind of annoying to have CI infrastructure changes like > this that differ between branches. Thoughts?
I am sharing patches per branch until we come up with a better solution. Some remarks: REL_15: It doesn't have any manual tasks, so per repository trigger patch (Thomas' patch) is omitted. REL_16: Only Mingw task is manual. REL_17: Only Mingw task is manual. While rebasing them, I realized that we use 'REPO_CI_AUTOMATIC_TRIGGER_TASKS' for all branches. So, we don't have per branch 'REPO_CI_AUTOMATIC_TRIGGER_TASKS' configuration option. Is that a problem? -- Regards, Nazir Bilal Yavuz Microsoft
From c872f537375416289d3fdc302716403266899c1f Mon Sep 17 00:00:00 2001 From: Andres Freund <and...@anarazel.de> Date: Fri, 12 Apr 2024 18:18:34 -0700 Subject: [PATCH v5] ci: Simplify ci-os-only handling Handle 'ci-os-only' occurrences in the .cirrus.star file instead of .cirrus.tasks.yml file. Now, 'ci-os-only' occurrences are controlled from one central place instead of dealing with them in each task. Author: Andres Freund <and...@anarazel.de> Reviewed-by: Nazir Bilal Yavuz <byavu...@gmail.com> Discussion: https://postgr.es/m/20240413021221.hg53rvqlvldqh57i%40awork3.anarazel.de Backpatch: 15-, where CI support was added --- .cirrus.star | 52 +++++++++++++++++++++++++++++++++++++++++++---- .cirrus.tasks.yml | 10 ++++----- .cirrus.yml | 12 +++++++++-- 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/.cirrus.star b/.cirrus.star index d2d6ceca207..3264ce014c5 100644 --- a/.cirrus.star +++ b/.cirrus.star @@ -7,7 +7,7 @@ https://github.com/bazelbuild/starlark/blob/master/spec.md See also .cirrus.yml and src/tools/ci/README """ -load("cirrus", "env", "fs") +load("cirrus", "env", "fs", "re", "yaml") def main(): @@ -18,19 +18,36 @@ def main(): 1) the contents of .cirrus.yml - 2) if defined, the contents of the file referenced by the, repository + 2) computed environment variables + + 3) if defined, the contents of the file referenced by the, repository level, REPO_CI_CONFIG_GIT_URL variable (see https://cirrus-ci.org/guide/programming-tasks/#fs for the accepted format) - 3) .cirrus.tasks.yml + 4) .cirrus.tasks.yml """ output = "" # 1) is evaluated implicitly + # Add 2) + additional_env = compute_environment_vars() + env_fmt = """ +### +# Computed environment variables start here +### +{0} +### +# Computed environment variables end here +### +""" + output += env_fmt.format(yaml.dumps({'env': additional_env})) + + + # Add 3) repo_config_url = env.get("REPO_CI_CONFIG_GIT_URL") if repo_config_url != None: print("loading additional configuration from \"{}\"".format(repo_config_url)) @@ -38,12 +55,39 @@ def main(): else: output += "\n# REPO_CI_CONFIG_URL was not set\n" - # Add 3) + + # Add 4) output += config_from(".cirrus.tasks.yml") + return output +def compute_environment_vars(): + cenv = {} + + # Parse "ci-os-only:" tag in commit message and set + # CI_{$OS}_ENABLED variable for each OS + + operating_systems = ['freebsd', 'linux', 'macos', 'windows', 'compilerwarnings'] + commit_message = env.get('CIRRUS_CHANGE_MESSAGE') + match_re = r"(^|.*\n)ci-os-only: ([^\n]+)($|\n.*)" + + # re.match() returns an array with a tuple of (matched-string, match_1, ...) + m = re.match(match_re, commit_message) + if m and len(m) > 0: + os_only = m[0][2] + os_only_list = re.split(r'[, ]+', os_only) + else: + os_only_list = operating_systems + + for os in operating_systems: + os_enabled = os in os_only_list + cenv['CI_{0}_ENABLED'.format(os.upper())] = os_enabled + + return cenv + + def config_from(config_src): """return contents of config file `config_src`, surrounded by markers indicating start / end of the the included file diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index 995ae7cefb9..ecf683b7b54 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -47,7 +47,7 @@ task: <<: *freebsd_task_template - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*' + only_if: $CI_FREEBSD_ENABLED sysinfo_script: | id @@ -153,7 +153,7 @@ task: <<: *linux_task_template - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*' + only_if: $CI_LINUX_ENABLED ccache_cache: folder: ${CCACHE_DIR} @@ -239,7 +239,7 @@ task: <<: *macos_task_template - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*' + only_if: $CI_MACOS_ENABLED sysinfo_script: | id @@ -389,7 +389,7 @@ task: <<: *windows_task_template - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*' + only_if: $CI_WINDOWS_ENABLED sysinfo_script: | chcp @@ -476,7 +476,7 @@ task: # task that did not run, count as a success, so we need to recheck Linux' # condition here ... - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*' + only_if: $CI_COMPILERWARNINGS_ENABLED <<: *linux_task_template diff --git a/.cirrus.yml b/.cirrus.yml index a83129ae46d..f270f61241f 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -10,12 +10,20 @@ # # 1) the contents of this file # -# 2) if defined, the contents of the file referenced by the, repository +# 2) computed environment variables +# +# Used to enable/disable tasks based on the execution environment. See +# .cirrus.star: compute_environment_vars() +# +# 3) if defined, the contents of the file referenced by the, repository # level, REPO_CI_CONFIG_GIT_URL variable (see # https://cirrus-ci.org/guide/programming-tasks/#fs for the accepted # format) # -# 3) .cirrus.tasks.yml +# This allows running tasks in a different execution environment than the +# default, e.g. to have sufficient resources for cfbot. +# +# 4) .cirrus.tasks.yml # # This composition is done by .cirrus.star -- 2.49.0
From b9276d7cc5713008a986c68d2f00b5bf23cb50fd Mon Sep 17 00:00:00 2001 From: Andres Freund <and...@anarazel.de> Date: Fri, 12 Apr 2024 15:04:32 -0700 Subject: [PATCH v5 1/2] ci: Per-repo triggers for not automated tasks We're not ready to trigger some tasks automatically by default, so you normally have to trigger then manually via Cirrus if you want to run them. This avoids using too many credits by default. With this commit, an individual repository can be configured to trigger them automatically using an environment variable defined under "Repository Settings", for example: REPO_CI_AUTOMATIC_TRIGGER_TASKS="mingw" This will enable cfbot to turn them on by default when running tests for the Commitfest app. It is not subject to free credit limits as it runs on credits donated by Google. Author: Andres Freund <and...@anarazel.de> Co-authored-by: Thomas Munro <thomas.mu...@gmail.com> Reviewed-by: Nazir Bilal Yavuz <byavu...@gmail.com> Discussion: https://postgr.es/m/20240413021221.hg53rvqlvldqh57i%40awork3.anarazel.de Backpatch: 16-, where first manual task was added --- .cirrus.star | 50 +++++++++++++++++++++++++++++++++++++++++---- .cirrus.tasks.yml | 10 ++++----- .cirrus.yml | 12 +++++++++-- src/tools/ci/README | 11 ++++++++++ 4 files changed, 71 insertions(+), 12 deletions(-) diff --git a/.cirrus.star b/.cirrus.star index d2d6ceca207..ff31d191565 100644 --- a/.cirrus.star +++ b/.cirrus.star @@ -7,7 +7,7 @@ https://github.com/bazelbuild/starlark/blob/master/spec.md See also .cirrus.yml and src/tools/ci/README """ -load("cirrus", "env", "fs") +load("cirrus", "env", "fs", "yaml") def main(): @@ -18,19 +18,36 @@ def main(): 1) the contents of .cirrus.yml - 2) if defined, the contents of the file referenced by the, repository + 2) computed environment variables + + 3) if defined, the contents of the file referenced by the, repository level, REPO_CI_CONFIG_GIT_URL variable (see https://cirrus-ci.org/guide/programming-tasks/#fs for the accepted format) - 3) .cirrus.tasks.yml + 4) .cirrus.tasks.yml """ output = "" # 1) is evaluated implicitly + # Add 2) + additional_env = compute_environment_vars() + env_fmt = """ +### +# Computed environment variables start here +### +{0} +### +# Computed environment variables end here +### +""" + output += env_fmt.format(yaml.dumps({'env': additional_env})) + + + # Add 3) repo_config_url = env.get("REPO_CI_CONFIG_GIT_URL") if repo_config_url != None: print("loading additional configuration from \"{}\"".format(repo_config_url)) @@ -38,12 +55,37 @@ def main(): else: output += "\n# REPO_CI_CONFIG_URL was not set\n" - # Add 3) + + # Add 4) output += config_from(".cirrus.tasks.yml") + return output +def compute_environment_vars(): + cenv = {} + + # The mingw task is manually triggered by default because it might use too + # many resources for users of free Cirrus credits, but you can trigger it + # automatically by naming it in an environment variable eg + # REPO_CI_AUTOMATIC_TRIGGER_TASKS="mingw" under "Repository Settings" on + # Cirrus CI's website. + + default_manual_trigger_tasks = ['mingw'] + + repo_ci_automatic_trigger_tasks = env.get('REPO_CI_AUTOMATIC_TRIGGER_TASKS', '') + for task in default_manual_trigger_tasks: + name = 'CI_TRIGGER_TYPE_' + task.upper() + if repo_ci_automatic_trigger_tasks.find(task) != -1: + value = 'automatic' + else: + value = 'manual' + cenv[name] = value + + return cenv + + def config_from(config_src): """return contents of config file `config_src`, surrounded by markers indicating start / end of the the included file diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index d12360aa356..54c6f3a65ee 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -591,13 +591,11 @@ task: << : *WINDOWS_ENVIRONMENT_BASE name: Windows - Server 2019, MinGW64 - Meson - # due to resource constraints we don't run this task by default for now - trigger_type: manual - # worth using only_if despite being manual, otherwise this task will show up - # when e.g. ci-os-only: linux is used. - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*mingw.*' - # otherwise it'll be sorted before other tasks + # See REPO_CI_AUTOMATIC_TRIGGER_TASKS in .cirrus.star. + trigger_type: $CI_TRIGGER_TYPE_MINGW + depends_on: SanityCheck + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*mingw.*' env: TEST_JOBS: 4 # higher concurrency causes occasional failures diff --git a/.cirrus.yml b/.cirrus.yml index a83129ae46d..f270f61241f 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -10,12 +10,20 @@ # # 1) the contents of this file # -# 2) if defined, the contents of the file referenced by the, repository +# 2) computed environment variables +# +# Used to enable/disable tasks based on the execution environment. See +# .cirrus.star: compute_environment_vars() +# +# 3) if defined, the contents of the file referenced by the, repository # level, REPO_CI_CONFIG_GIT_URL variable (see # https://cirrus-ci.org/guide/programming-tasks/#fs for the accepted # format) # -# 3) .cirrus.tasks.yml +# This allows running tasks in a different execution environment than the +# default, e.g. to have sufficient resources for cfbot. +# +# 4) .cirrus.tasks.yml # # This composition is done by .cirrus.star diff --git a/src/tools/ci/README b/src/tools/ci/README index 30ddd200c96..771243c4da0 100644 --- a/src/tools/ci/README +++ b/src/tools/ci/README @@ -82,3 +82,14 @@ defined in .cirrus.yml, by redefining the relevant yaml anchors. Custom compute resources can be provided using - https://cirrus-ci.org/guide/supported-computing-services/ - https://cirrus-ci.org/guide/persistent-workers/ + + +Enabling manual tasks by default +================================ + +Some tasks are not triggered automatically by default, to avoid using up CI +credits too quickly. This can be changed on the repository level, e.g. when +custom compute resources are configured. + +The following repository level environment variables are recognized: +- REPO_CI_AUTOMATIC_TRIGGER_TASKS="mingw" -- 2.49.0
From 5975233e10c8bb4e87049cccb13c98cbdc342532 Mon Sep 17 00:00:00 2001 From: Andres Freund <and...@anarazel.de> Date: Fri, 12 Apr 2024 18:18:34 -0700 Subject: [PATCH v5 2/2] ci: Simplify ci-os-only handling Handle 'ci-os-only' occurrences in the .cirrus.star file instead of .cirrus.tasks.yml file. Now, 'ci-os-only' occurrences are controlled from one central place instead of dealing with them in each task. Author: Andres Freund <and...@anarazel.de> Reviewed-by: Nazir Bilal Yavuz <byavu...@gmail.com> Discussion: https://postgr.es/m/20240413021221.hg53rvqlvldqh57i%40awork3.anarazel.de Backpatch: 15-, where CI support was added --- .cirrus.star | 31 ++++++++++++++++++++++++++++++- .cirrus.tasks.yml | 17 ++++++++--------- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/.cirrus.star b/.cirrus.star index ff31d191565..06666e55a4b 100644 --- a/.cirrus.star +++ b/.cirrus.star @@ -7,7 +7,7 @@ https://github.com/bazelbuild/starlark/blob/master/spec.md See also .cirrus.yml and src/tools/ci/README """ -load("cirrus", "env", "fs", "yaml") +load("cirrus", "env", "fs", "re", "yaml") def main(): @@ -66,6 +66,7 @@ def main(): def compute_environment_vars(): cenv = {} + ### # The mingw task is manually triggered by default because it might use too # many resources for users of free Cirrus credits, but you can trigger it # automatically by naming it in an environment variable eg @@ -82,6 +83,34 @@ def compute_environment_vars(): else: value = 'manual' cenv[name] = value + ### + + ### + # Parse "ci-os-only:" tag in commit message and set + # CI_{$OS}_ENABLED variable for each OS + + # We want to disable SanityCheck if testing just a specific OS. This + # shortens push-wait-for-ci cycle time a bit when debugging operating + # system specific failures. Just treating it as an OS in that case + # suffices. + + operating_systems = ['freebsd', 'linux', 'macos', 'windows', 'mingw', + 'sanitycheck', 'compilerwarnings'] + commit_message = env.get('CIRRUS_CHANGE_MESSAGE') + match_re = r"(^|.*\n)ci-os-only: ([^\n]+)($|\n.*)" + + # re.match() returns an array with a tuple of (matched-string, match_1, ...) + m = re.match(match_re, commit_message) + if m and len(m) > 0: + os_only = m[0][2] + os_only_list = re.split(r'[, ]+', os_only) + else: + os_only_list = operating_systems + + for os in operating_systems: + os_enabled = os in os_only_list + cenv['CI_{0}_ENABLED'.format(os.upper())] = os_enabled + ### return cenv diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index 54c6f3a65ee..59d69acf2a7 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -62,7 +62,7 @@ task: # push-wait-for-ci cycle time a bit when debugging operating system specific # failures. Uses skip instead of only_if, as cirrus otherwise warns about # only_if conditions not matching. - skip: $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:.*' + skip: $CI_SANITYCHECK_ENABLED == false env: CPUS: 4 @@ -143,7 +143,7 @@ task: <<: *freebsd_task_template depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*' + only_if: $CI_FREEBSD_ENABLED sysinfo_script: | id @@ -280,7 +280,7 @@ task: <<: *linux_task_template depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*' + only_if: $CI_LINUX_ENABLED ccache_cache: folder: ${CCACHE_DIR} @@ -449,7 +449,7 @@ task: <<: *macos_task_template depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*' + only_if: $CI_MACOS_ENABLED sysinfo_script: | id @@ -556,7 +556,7 @@ task: <<: *windows_task_template depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*' + only_if: $CI_WINDOWS_ENABLED setup_additional_packages_script: | REM choco install -y --no-progress ... @@ -595,7 +595,7 @@ task: trigger_type: $CI_TRIGGER_TYPE_MINGW depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*mingw.*' + only_if: $CI_MINGW_ENABLED env: TEST_JOBS: 4 # higher concurrency causes occasional failures @@ -649,10 +649,9 @@ task: # To limit unnecessary work only run this once the SanityCheck # succeeds. This is particularly important for this task as we intentionally - # use always: to continue after failures. Task that did not run count as a - # success, so we need to recheck SanityChecks's condition here ... + # use always: to continue after failures. depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' + only_if: $CI_COMPILERWARNINGS_ENABLED env: CPUS: 4 -- 2.49.0
From 340782a5e770b85d05f4e583d6dc1f12b1888a96 Mon Sep 17 00:00:00 2001 From: Andres Freund <and...@anarazel.de> Date: Fri, 12 Apr 2024 15:04:32 -0700 Subject: [PATCH v5 1/2] ci: Per-repo triggers for not automated tasks We're not ready to trigger some tasks automatically by default, so you normally have to trigger then manually via Cirrus if you want to run them. This avoids using too many credits by default. With this commit, an individual repository can be configured to trigger them automatically using an environment variable defined under "Repository Settings", for example: REPO_CI_AUTOMATIC_TRIGGER_TASKS="mingw" This will enable cfbot to turn them on by default when running tests for the Commitfest app. It is not subject to free credit limits as it runs on credits donated by Google. Author: Andres Freund <and...@anarazel.de> Co-authored-by: Thomas Munro <thomas.mu...@gmail.com> Reviewed-by: Nazir Bilal Yavuz <byavu...@gmail.com> Discussion: https://postgr.es/m/20240413021221.hg53rvqlvldqh57i%40awork3.anarazel.de Backpatch: 16-, where first manual task was added --- .cirrus.star | 50 +++++++++++++++++++++++++++++++++++++++++---- .cirrus.tasks.yml | 10 ++++----- .cirrus.yml | 12 +++++++++-- src/tools/ci/README | 11 ++++++++++ 4 files changed, 71 insertions(+), 12 deletions(-) diff --git a/.cirrus.star b/.cirrus.star index 36233872d1e..cccdbd1bfbb 100644 --- a/.cirrus.star +++ b/.cirrus.star @@ -7,7 +7,7 @@ https://github.com/bazelbuild/starlark/blob/master/spec.md See also .cirrus.yml and src/tools/ci/README """ -load("cirrus", "env", "fs") +load("cirrus", "env", "fs", "yaml") def main(): @@ -18,19 +18,36 @@ def main(): 1) the contents of .cirrus.yml - 2) if defined, the contents of the file referenced by the, repository + 2) computed environment variables + + 3) if defined, the contents of the file referenced by the, repository level, REPO_CI_CONFIG_GIT_URL variable (see https://cirrus-ci.org/guide/programming-tasks/#fs for the accepted format) - 3) .cirrus.tasks.yml + 4) .cirrus.tasks.yml """ output = "" # 1) is evaluated implicitly + # Add 2) + additional_env = compute_environment_vars() + env_fmt = """ +### +# Computed environment variables start here +### +{0} +### +# Computed environment variables end here +### +""" + output += env_fmt.format(yaml.dumps({'env': additional_env})) + + + # Add 3) repo_config_url = env.get("REPO_CI_CONFIG_GIT_URL") if repo_config_url != None: print("loading additional configuration from \"{}\"".format(repo_config_url)) @@ -38,12 +55,37 @@ def main(): else: output += "\n# REPO_CI_CONFIG_URL was not set\n" - # Add 3) + + # Add 4) output += config_from(".cirrus.tasks.yml") + return output +def compute_environment_vars(): + cenv = {} + + # The mingw task is manually triggered by default because it might use too + # many resources for users of free Cirrus credits, but you can trigger it + # automatically by naming it in an environment variable eg + # REPO_CI_AUTOMATIC_TRIGGER_TASKS="mingw" under "Repository Settings" on + # Cirrus CI's website. + + default_manual_trigger_tasks = ['mingw'] + + repo_ci_automatic_trigger_tasks = env.get('REPO_CI_AUTOMATIC_TRIGGER_TASKS', '') + for task in default_manual_trigger_tasks: + name = 'CI_TRIGGER_TYPE_' + task.upper() + if repo_ci_automatic_trigger_tasks.find(task) != -1: + value = 'automatic' + else: + value = 'manual' + cenv[name] = value + + return cenv + + def config_from(config_src): """return contents of config file `config_src`, surrounded by markers indicating start / end of the included file diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index aa7c7dcd34d..af33716a004 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -595,13 +595,11 @@ task: << : *WINDOWS_ENVIRONMENT_BASE name: Windows - Server 2019, MinGW64 - Meson - # due to resource constraints we don't run this task by default for now - trigger_type: manual - # worth using only_if despite being manual, otherwise this task will show up - # when e.g. ci-os-only: linux is used. - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*mingw.*' - # otherwise it'll be sorted before other tasks + # See REPO_CI_AUTOMATIC_TRIGGER_TASKS in .cirrus.star. + trigger_type: $CI_TRIGGER_TYPE_MINGW + depends_on: SanityCheck + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*mingw.*' env: TEST_JOBS: 4 # higher concurrency causes occasional failures diff --git a/.cirrus.yml b/.cirrus.yml index a83129ae46d..f270f61241f 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -10,12 +10,20 @@ # # 1) the contents of this file # -# 2) if defined, the contents of the file referenced by the, repository +# 2) computed environment variables +# +# Used to enable/disable tasks based on the execution environment. See +# .cirrus.star: compute_environment_vars() +# +# 3) if defined, the contents of the file referenced by the, repository # level, REPO_CI_CONFIG_GIT_URL variable (see # https://cirrus-ci.org/guide/programming-tasks/#fs for the accepted # format) # -# 3) .cirrus.tasks.yml +# This allows running tasks in a different execution environment than the +# default, e.g. to have sufficient resources for cfbot. +# +# 4) .cirrus.tasks.yml # # This composition is done by .cirrus.star diff --git a/src/tools/ci/README b/src/tools/ci/README index 30ddd200c96..771243c4da0 100644 --- a/src/tools/ci/README +++ b/src/tools/ci/README @@ -82,3 +82,14 @@ defined in .cirrus.yml, by redefining the relevant yaml anchors. Custom compute resources can be provided using - https://cirrus-ci.org/guide/supported-computing-services/ - https://cirrus-ci.org/guide/persistent-workers/ + + +Enabling manual tasks by default +================================ + +Some tasks are not triggered automatically by default, to avoid using up CI +credits too quickly. This can be changed on the repository level, e.g. when +custom compute resources are configured. + +The following repository level environment variables are recognized: +- REPO_CI_AUTOMATIC_TRIGGER_TASKS="mingw" -- 2.49.0
From 903489cc02b40f44e3672dda2fdbcd9dd7147056 Mon Sep 17 00:00:00 2001 From: Andres Freund <and...@anarazel.de> Date: Fri, 12 Apr 2024 18:18:34 -0700 Subject: [PATCH v5 2/2] ci: Simplify ci-os-only handling Handle 'ci-os-only' occurrences in the .cirrus.star file instead of .cirrus.tasks.yml file. Now, 'ci-os-only' occurrences are controlled from one central place instead of dealing with them in each task. Author: Andres Freund <and...@anarazel.de> Reviewed-by: Nazir Bilal Yavuz <byavu...@gmail.com> Discussion: https://postgr.es/m/20240413021221.hg53rvqlvldqh57i%40awork3.anarazel.de Backpatch: 15-, where CI support was added --- .cirrus.star | 31 ++++++++++++++++++++++++++++++- .cirrus.tasks.yml | 17 ++++++++--------- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/.cirrus.star b/.cirrus.star index cccdbd1bfbb..c26cf52d10c 100644 --- a/.cirrus.star +++ b/.cirrus.star @@ -7,7 +7,7 @@ https://github.com/bazelbuild/starlark/blob/master/spec.md See also .cirrus.yml and src/tools/ci/README """ -load("cirrus", "env", "fs", "yaml") +load("cirrus", "env", "fs", "re", "yaml") def main(): @@ -66,6 +66,7 @@ def main(): def compute_environment_vars(): cenv = {} + ### # The mingw task is manually triggered by default because it might use too # many resources for users of free Cirrus credits, but you can trigger it # automatically by naming it in an environment variable eg @@ -82,6 +83,34 @@ def compute_environment_vars(): else: value = 'manual' cenv[name] = value + ### + + ### + # Parse "ci-os-only:" tag in commit message and set + # CI_{$OS}_ENABLED variable for each OS + + # We want to disable SanityCheck if testing just a specific OS. This + # shortens push-wait-for-ci cycle time a bit when debugging operating + # system specific failures. Just treating it as an OS in that case + # suffices. + + operating_systems = ['freebsd', 'linux', 'macos', 'windows', 'mingw', + 'sanitycheck', 'compilerwarnings'] + commit_message = env.get('CIRRUS_CHANGE_MESSAGE') + match_re = r"(^|.*\n)ci-os-only: ([^\n]+)($|\n.*)" + + # re.match() returns an array with a tuple of (matched-string, match_1, ...) + m = re.match(match_re, commit_message) + if m and len(m) > 0: + os_only = m[0][2] + os_only_list = re.split(r'[, ]+', os_only) + else: + os_only_list = operating_systems + + for os in operating_systems: + os_enabled = os in os_only_list + cenv['CI_{0}_ENABLED'.format(os.upper())] = os_enabled + ### return cenv diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index af33716a004..93459e808f0 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -62,7 +62,7 @@ task: # push-wait-for-ci cycle time a bit when debugging operating system specific # failures. Uses skip instead of only_if, as cirrus otherwise warns about # only_if conditions not matching. - skip: $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:.*' + skip: $CI_SANITYCHECK_ENABLED == false env: CPUS: 4 @@ -144,7 +144,7 @@ task: <<: *freebsd_task_template depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*' + only_if: $CI_FREEBSD_ENABLED sysinfo_script: | id @@ -282,7 +282,7 @@ task: <<: *linux_task_template depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*' + only_if: $CI_LINUX_ENABLED ccache_cache: folder: ${CCACHE_DIR} @@ -453,7 +453,7 @@ task: <<: *macos_task_template depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*' + only_if: $CI_MACOS_ENABLED sysinfo_script: | id @@ -560,7 +560,7 @@ task: <<: *windows_task_template depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*' + only_if: $CI_WINDOWS_ENABLED setup_additional_packages_script: | REM choco install -y --no-progress ... @@ -599,7 +599,7 @@ task: trigger_type: $CI_TRIGGER_TYPE_MINGW depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*mingw.*' + only_if: $CI_MINGW_ENABLED env: TEST_JOBS: 4 # higher concurrency causes occasional failures @@ -653,10 +653,9 @@ task: # To limit unnecessary work only run this once the SanityCheck # succeeds. This is particularly important for this task as we intentionally - # use always: to continue after failures. Task that did not run count as a - # success, so we need to recheck SanityChecks's condition here ... + # use always: to continue after failures. depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' + only_if: $CI_COMPILERWARNINGS_ENABLED env: CPUS: 4 -- 2.49.0
From c50fdccc3b3e1aee9f51c576da5942bcc1732e14 Mon Sep 17 00:00:00 2001 From: Andres Freund <and...@anarazel.de> Date: Fri, 12 Apr 2024 15:04:32 -0700 Subject: [PATCH v5 1/2] ci: Per-repo triggers for not automated tasks We're not ready to trigger some tasks automatically by default, so you normally have to trigger then manually via Cirrus if you want to run them. This avoids using too many credits by default. With this commit, an individual repository can be configured to trigger them automatically using an environment variable defined under "Repository Settings", for example: REPO_CI_AUTOMATIC_TRIGGER_TASKS="mingw netbsd openbsd" This will enable cfbot to turn them on by default when running tests for the Commitfest app. It is not subject to free credit limits as it runs on credits donated by Google. Author: Andres Freund <and...@anarazel.de> Co-authored-by: Thomas Munro <thomas.mu...@gmail.com> Reviewed-by: Nazir Bilal Yavuz <byavu...@gmail.com> Discussion: https://postgr.es/m/20240413021221.hg53rvqlvldqh57i%40awork3.anarazel.de Backpatch: 16-, where first manual task was added --- .cirrus.star | 50 +++++++++++++++++++++++++++++++++++++++++---- .cirrus.tasks.yml | 15 +++++++------- .cirrus.yml | 12 +++++++++-- src/tools/ci/README | 11 ++++++++++ 4 files changed, 75 insertions(+), 13 deletions(-) diff --git a/.cirrus.star b/.cirrus.star index 36233872d1e..218fce887b5 100644 --- a/.cirrus.star +++ b/.cirrus.star @@ -7,7 +7,7 @@ https://github.com/bazelbuild/starlark/blob/master/spec.md See also .cirrus.yml and src/tools/ci/README """ -load("cirrus", "env", "fs") +load("cirrus", "env", "fs", "yaml") def main(): @@ -18,19 +18,36 @@ def main(): 1) the contents of .cirrus.yml - 2) if defined, the contents of the file referenced by the, repository + 2) computed environment variables + + 3) if defined, the contents of the file referenced by the, repository level, REPO_CI_CONFIG_GIT_URL variable (see https://cirrus-ci.org/guide/programming-tasks/#fs for the accepted format) - 3) .cirrus.tasks.yml + 4) .cirrus.tasks.yml """ output = "" # 1) is evaluated implicitly + # Add 2) + additional_env = compute_environment_vars() + env_fmt = """ +### +# Computed environment variables start here +### +{0} +### +# Computed environment variables end here +### +""" + output += env_fmt.format(yaml.dumps({'env': additional_env})) + + + # Add 3) repo_config_url = env.get("REPO_CI_CONFIG_GIT_URL") if repo_config_url != None: print("loading additional configuration from \"{}\"".format(repo_config_url)) @@ -38,12 +55,37 @@ def main(): else: output += "\n# REPO_CI_CONFIG_URL was not set\n" - # Add 3) + + # Add 4) output += config_from(".cirrus.tasks.yml") + return output +def compute_environment_vars(): + cenv = {} + + # The following tasks are manually triggered by default because they + # might use too many resources for users of free Cirrus credits, but you can + # trigger them automatically by naming them in an environment variable eg + # REPO_CI_AUTOMATIC_TRIGGER_TASKS="mingw netbsd" under "Repository + # Settings" on Cirrus CI's website. + + default_manual_trigger_tasks = ['mingw', 'netbsd', 'openbsd'] + + repo_ci_automatic_trigger_tasks = env.get('REPO_CI_AUTOMATIC_TRIGGER_TASKS', '') + for task in default_manual_trigger_tasks: + name = 'CI_TRIGGER_TYPE_' + task.upper() + if repo_ci_automatic_trigger_tasks.find(task) != -1: + value = 'automatic' + else: + value = 'manual' + cenv[name] = value + + return cenv + + def config_from(config_src): """return contents of config file `config_src`, surrounded by markers indicating start / end of the included file diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index 86a1fa9bbdb..0cf6ea67ba5 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -239,7 +239,6 @@ task: task: depends_on: SanityCheck - trigger_type: manual env: # Below are experimentally derived to be a decent choice. @@ -257,6 +256,8 @@ task: matrix: - name: NetBSD - Meson + # See REPO_CI_AUTOMATIC_TRIGGER_TASKS in .cirrus.star + trigger_type: $CI_TRIGGER_TYPE_NETBSD only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*netbsd.*' env: OS_NAME: netbsd @@ -274,6 +275,8 @@ task: <<: *netbsd_task_template - name: OpenBSD - Meson + # See REPO_CI_AUTOMATIC_TRIGGER_TASKS in .cirrus.star + trigger_type: $CI_TRIGGER_TYPE_OPENBSD only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*openbsd.*' env: OS_NAME: openbsd @@ -743,13 +746,11 @@ task: << : *WINDOWS_ENVIRONMENT_BASE name: Windows - Server 2019, MinGW64 - Meson - # due to resource constraints we don't run this task by default for now - trigger_type: manual - # worth using only_if despite being manual, otherwise this task will show up - # when e.g. ci-os-only: linux is used. - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*mingw.*' - # otherwise it'll be sorted before other tasks + # See REPO_CI_AUTOMATIC_TRIGGER_TASKS in .cirrus.star. + trigger_type: $CI_TRIGGER_TYPE_MINGW + depends_on: SanityCheck + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*mingw.*' env: TEST_JOBS: 4 # higher concurrency causes occasional failures diff --git a/.cirrus.yml b/.cirrus.yml index 33c6e481d74..3f75852e84e 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -10,12 +10,20 @@ # # 1) the contents of this file # -# 2) if defined, the contents of the file referenced by the, repository +# 2) computed environment variables +# +# Used to enable/disable tasks based on the execution environment. See +# .cirrus.star: compute_environment_vars() +# +# 3) if defined, the contents of the file referenced by the, repository # level, REPO_CI_CONFIG_GIT_URL variable (see # https://cirrus-ci.org/guide/programming-tasks/#fs for the accepted # format) # -# 3) .cirrus.tasks.yml +# This allows running tasks in a different execution environment than the +# default, e.g. to have sufficient resources for cfbot. +# +# 4) .cirrus.tasks.yml # # This composition is done by .cirrus.star diff --git a/src/tools/ci/README b/src/tools/ci/README index 12c1e7c308f..d183648a8d0 100644 --- a/src/tools/ci/README +++ b/src/tools/ci/README @@ -82,3 +82,14 @@ defined in .cirrus.yml, by redefining the relevant yaml anchors. Custom compute resources can be provided using - https://cirrus-ci.org/guide/supported-computing-services/ - https://cirrus-ci.org/guide/persistent-workers/ + + +Enabling manual tasks by default +================================ + +Some tasks are not triggered automatically by default, to avoid using up CI +credits too quickly. This can be changed on the repository level, e.g. when +custom compute resources are configured. + +The following repository level environment variables are recognized: +- REPO_CI_AUTOMATIC_TRIGGER_TASKS - space-separated list of (mingw|netbsd|openbsd) -- 2.49.0
From bcd64a5fb6d6721fceba1684a3bfc0b46c7fd9d5 Mon Sep 17 00:00:00 2001 From: Andres Freund <and...@anarazel.de> Date: Fri, 12 Apr 2024 18:18:34 -0700 Subject: [PATCH v5 2/2] ci: Simplify ci-os-only handling Handle 'ci-os-only' occurrences in the .cirrus.star file instead of .cirrus.tasks.yml file. Now, 'ci-os-only' occurrences are controlled from one central place instead of dealing with them in each task. Author: Andres Freund <and...@anarazel.de> Reviewed-by: Nazir Bilal Yavuz <byavu...@gmail.com> Discussion: https://postgr.es/m/20240413021221.hg53rvqlvldqh57i%40awork3.anarazel.de Backpatch: 15-, where CI support was added --- .cirrus.star | 31 ++++++++++++++++++++++++++++++- .cirrus.tasks.yml | 21 ++++++++++----------- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/.cirrus.star b/.cirrus.star index 218fce887b5..83e93e48d5f 100644 --- a/.cirrus.star +++ b/.cirrus.star @@ -7,7 +7,7 @@ https://github.com/bazelbuild/starlark/blob/master/spec.md See also .cirrus.yml and src/tools/ci/README """ -load("cirrus", "env", "fs", "yaml") +load("cirrus", "env", "fs", "re", "yaml") def main(): @@ -66,6 +66,7 @@ def main(): def compute_environment_vars(): cenv = {} + ### # The following tasks are manually triggered by default because they # might use too many resources for users of free Cirrus credits, but you can # trigger them automatically by naming them in an environment variable eg @@ -82,6 +83,34 @@ def compute_environment_vars(): else: value = 'manual' cenv[name] = value + ### + + ### + # Parse "ci-os-only:" tag in commit message and set + # CI_{$OS}_ENABLED variable for each OS + + # We want to disable SanityCheck if testing just a specific OS. This + # shortens push-wait-for-ci cycle time a bit when debugging operating + # system specific failures. Just treating it as an OS in that case + # suffices. + + operating_systems = ['freebsd', 'netbsd', 'openbsd', 'linux', 'macos', + 'windows', 'mingw', 'sanitycheck', 'compilerwarnings'] + commit_message = env.get('CIRRUS_CHANGE_MESSAGE') + match_re = r"(^|.*\n)ci-os-only: ([^\n]+)($|\n.*)" + + # re.match() returns an array with a tuple of (matched-string, match_1, ...) + m = re.match(match_re, commit_message) + if m and len(m) > 0: + os_only = m[0][2] + os_only_list = re.split(r'[, ]+', os_only) + else: + os_only_list = operating_systems + + for os in operating_systems: + os_enabled = os in os_only_list + cenv['CI_{0}_ENABLED'.format(os.upper())] = os_enabled + ### return cenv diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index 0cf6ea67ba5..744b1255ed9 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -72,7 +72,7 @@ task: # push-wait-for-ci cycle time a bit when debugging operating system specific # failures. Uses skip instead of only_if, as cirrus otherwise warns about # only_if conditions not matching. - skip: $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:.*' + skip: $CI_SANITYCHECK_ENABLED == false env: CPUS: 4 @@ -167,7 +167,7 @@ task: <<: *freebsd_task_template depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*' + only_if: $CI_FREEBSD_ENABLED sysinfo_script: | id @@ -258,7 +258,7 @@ task: - name: NetBSD - Meson # See REPO_CI_AUTOMATIC_TRIGGER_TASKS in .cirrus.star trigger_type: $CI_TRIGGER_TYPE_NETBSD - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*netbsd.*' + only_if: $CI_NETBSD_ENABLED env: OS_NAME: netbsd IMAGE_FAMILY: pg-ci-netbsd-postgres @@ -277,7 +277,7 @@ task: - name: OpenBSD - Meson # See REPO_CI_AUTOMATIC_TRIGGER_TASKS in .cirrus.star trigger_type: $CI_TRIGGER_TYPE_OPENBSD - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*openbsd.*' + only_if: $CI_OPENBSD_ENABLED env: OS_NAME: openbsd IMAGE_FAMILY: pg-ci-openbsd-postgres @@ -417,7 +417,7 @@ task: <<: *linux_task_template depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*' + only_if: $CI_LINUX_ENABLED ccache_cache: folder: ${CCACHE_DIR} @@ -605,7 +605,7 @@ task: <<: *macos_task_template depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*' + only_if: $CI_MACOS_ENABLED sysinfo_script: | id @@ -711,7 +711,7 @@ task: <<: *windows_task_template depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*' + only_if: $CI_WINDOWS_ENABLED setup_additional_packages_script: | REM choco install -y --no-progress ... @@ -750,7 +750,7 @@ task: trigger_type: $CI_TRIGGER_TYPE_MINGW depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*mingw.*' + only_if: $CI_MINGW_ENABLED env: TEST_JOBS: 4 # higher concurrency causes occasional failures @@ -804,10 +804,9 @@ task: # To limit unnecessary work only run this once the SanityCheck # succeeds. This is particularly important for this task as we intentionally - # use always: to continue after failures. Task that did not run count as a - # success, so we need to recheck SanityChecks's condition here ... + # use always: to continue after failures. depends_on: SanityCheck - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' + only_if: $CI_COMPILERWARNINGS_ENABLED env: CPUS: 4 -- 2.49.0
