This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch terraform-lock-all-platforms in repository https://gitbox.apache.org/repos/asf/airflow-ci-infra.git
commit b970471c1fc40990e3c8357ff246cc4b9bcd7fe4 Author: Jarek Potiuk <[email protected]> AuthorDate: Sat Oct 31 11:40:19 2026 +0100 Keep Terraform lock files usable on both Linux and macOS terraform init records the provider checksum of the platform it runs on only, so a lock file generated on a Mac fails the workflows' terraform init -lockfile=readonly on the Linux runners. The terraform/runs-on-flex lock file was nearly committed that way. Generated-by: Claude Opus 5 --- .pre-commit-config.yaml | 8 ++ AGENTS.md | 42 +++++++++ scripts/terraform_lock_platforms.py | 115 +++++++++++++++++++++++++ tests/scripts/test_terraform_lock_platforms.py | 88 +++++++++++++++++++ 4 files changed, 253 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d72d0a5..5a89551 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -103,3 +103,11 @@ repos: hooks: - id: zizmor name: Run zizmor on GitHub Actions workflows + - repo: local + hooks: + - id: terraform-lock-platforms + name: Check Terraform lock files cover Linux and macOS + entry: scripts/terraform_lock_platforms.py + language: python + files: ^terraform/[^/]+/(.+\.tf|\.terraform\.lock\.hcl)$ + require_serial: true diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..4d2bb53 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,42 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +# AGENTS instructions + +## Terraform lock files + +Every Terraform module under `terraform/` commits its `.terraform.lock.hcl`, and the lock file +must hold provider checksums for Linux (the CI runners) and macOS (maintainers' machines). The +workflows run `terraform init -lockfile=readonly`, which fails on a platform the lock file does +not cover. `terraform init` alone records only the platform it runs on. + +Whenever a pull request adds a module, adds a provider or changes a provider version, generate +the lock file for all platforms and commit it in the same pull request: + +```bash +cd terraform/<module> +terraform init -backend=false +terraform providers lock -platform=linux_amd64 -platform=darwin_arm64 -platform=darwin_amd64 +``` + +The `terraform-lock-platforms` prek hook (`scripts/terraform_lock_platforms.py`) checks this on +every commit that touches a module, and in the `prek` workflow on every pull request. Where +`terraform` is installed it runs the commands above itself and fails the commit, so you review and +commit the regenerated lock file. If `terraform` cannot reach the registry (in an agent sandbox, for +example), ask the user to run the commands; never hand-edit the checksums. diff --git a/scripts/terraform_lock_platforms.py b/scripts/terraform_lock_platforms.py new file mode 100755 index 0000000..cb2ff6b --- /dev/null +++ b/scripts/terraform_lock_platforms.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Keep every Terraform lock file usable on the CI runners and on maintainers' Macs. + +``terraform init`` records the package checksum (``h1:``) of the platform it runs on only, so a +lock file made on a Mac fails the workflows' ``terraform init -lockfile=readonly`` on Linux, and +the other way round. ``terraform providers lock`` with every platform below records one ``h1:`` +per platform. The lock file does not say which platform a checksum is for, so this hook counts +them, which catches a lock file written by a plain ``terraform init``; it also checks that each +provider of the module's own ``required_providers`` is locked. That keeps it offline, so it runs in +CI too. Providers that only nested modules require, and changed version constraints, are left to +the workflows' ``terraform init -lockfile=readonly``. Where ``terraform`` is installed, the hook +regenerates the lock files that fail and exits non-zero so the result is committed, as any fixing +hook does. +""" + +import re +import shutil +import subprocess +import sys +from pathlib import Path + +PLATFORMS = ["linux_amd64", "darwin_arm64", "darwin_amd64"] + +PROVIDER_BLOCK = re.compile(r'^provider "([^"]+)" \{(.*?)^\}', re.MULTILINE | re.DOTALL) +SOURCE = re.compile(r'source\s*=\s*"([^"]+)"') +# A comment, or a quoted string to step over so a "#" inside it does not start one. +COMMENT_OR_STRING = re.compile(r'"(?:[^"\\\n]|\\.)*"|#[^\n]*|//[^\n]*|/\*.*?\*/', re.DOTALL) + + +def strip_comments(text: str) -> str: + return COMMENT_OR_STRING.sub(lambda m: m.group() if m.group().startswith('"') else "", text) + + +def find_required_providers(directory: Path) -> set[str]: + providers = set() + for tf_file in directory.glob("*.tf"): + text = strip_comments(tf_file.read_text()) + for start in (match.end() for match in re.finditer(r"required_providers\s*\{", text)): + depth, end = 1, start + while depth and end < len(text): + depth += {"{": 1, "}": -1}.get(text[end], 0) + end += 1 + for source in SOURCE.findall(text[start:end]): + providers.add(source if source.count("/") == 2 else f"registry.terraform.io/{source}") + return providers + + +def find_problems(directory: Path) -> list[str]: + lock_file = directory / ".terraform.lock.hcl" + if not lock_file.exists(): + return [f"{lock_file} is missing"] + locked = dict(PROVIDER_BLOCK.findall(lock_file.read_text())) + problems = [ + f"{lock_file}: {provider} is required but not locked" + for provider in sorted(find_required_providers(directory) - locked.keys()) + ] + for provider, body in locked.items(): + h1_count = body.count('"h1:') + if h1_count < len(PLATFORMS): + problems.append( + f"{lock_file}: {provider} has {h1_count} h1: checksum(s), expected one for each of " + f"{', '.join(PLATFORMS)}" + ) + return problems + + +def regenerate(directory: Path) -> bool: + platform_args = [f"-platform={platform}" for platform in PLATFORMS] + for command in ( + # providers lock needs the modules installed to see their provider requirements. + ["terraform", "init", "-backend=false", "-input=false"], + ["terraform", "providers", "lock", *platform_args], + ): + if subprocess.run(command, cwd=directory).returncode != 0: + return False + return True + + +def main(filenames: list[str]) -> int: + directories = sorted({Path(name).parent for name in filenames}) + failed = False + for directory in directories: + problems = find_problems(directory) + if not problems: + continue + failed = True + print("\n".join(problems)) + if shutil.which("terraform") is None: + print( + f"Run in {directory}: terraform init -backend=false && terraform providers lock " + + " ".join(f"-platform={platform}" for platform in PLATFORMS) + ) + elif regenerate(directory): + print(f"Regenerated {directory / '.terraform.lock.hcl'}; review and commit it.") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/scripts/test_terraform_lock_platforms.py b/tests/scripts/test_terraform_lock_platforms.py new file mode 100644 index 0000000..df7d4eb --- /dev/null +++ b/tests/scripts/test_terraform_lock_platforms.py @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parents[2] / "scripts")) + +from terraform_lock_platforms import find_problems # noqa: E402 + +LOCK = """ +provider "registry.terraform.io/hashicorp/aws" { + version = "6.70.0" + hashes = [ +%s + "zh:068ca7ed4fc77e3d63d7fac54515a426a32f3dc143b55a6ad00963fe68289955", + ] +} +""" +THREE_PLATFORMS = ' "h1:a=",\n "h1:b=",\n "h1:c=",' +REQUIRES_AWS = """ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" # "#" in a comment + version = "~> 6.0" + } + # random = { source = "hashicorp/random" } + // time = { source = "hashicorp/time" } + /* null = { source = "hashicorp/null" } */ + } +} + +module "runs_on" { + source = "runs-on/runs-on/aws//modules/flex" +} +""" + + [email protected]( + "hashes, version_tf, expected", + [ + pytest.param(THREE_PLATFORMS, REQUIRES_AWS, [], id="complete"), + pytest.param( + ' "h1:a=",', + REQUIRES_AWS, + ["registry.terraform.io/hashicorp/aws has 1 h1: checksum(s)"], + id="one-platform", + ), + pytest.param( + THREE_PLATFORMS, + REQUIRES_AWS + 'terraform {\n required_providers {\n random = {\n' + ' source = "hashicorp/random"\n }\n }\n}\n', + ["registry.terraform.io/hashicorp/random is required but not locked"], + id="unlocked-provider", + ), + ], +) +def test_find_problems(tmp_path, hashes, version_tf, expected): + (tmp_path / ".terraform.lock.hcl").write_text(LOCK % hashes) + (tmp_path / "version.tf").write_text(version_tf) + + problems = find_problems(tmp_path) + + assert len(problems) == len(expected) + for problem, fragment in zip(problems, expected): + assert fragment in problem + + +def test_find_problems_without_lock_file(tmp_path): + (tmp_path / "version.tf").write_text(REQUIRES_AWS) + + assert find_problems(tmp_path) == [f"{tmp_path / '.terraform.lock.hcl'} is missing"]
