This is an automated email from the ASF dual-hosted git repository. acosentino pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/camel.git
commit 735e984551ee9676275ad7be84f4f88703f42b48 Author: Andrea Cosentino <[email protected]> AuthorDate: Tue Dec 2 09:39:32 2025 +0100 Attempt at Check container upgrade Signed-off-by: Andrea Cosentino <[email protected]> --- .../check-container-versions.py | 939 +++++++++++++++++++++ .github/workflows/check-container-versions.yml | 331 ++++++++ 2 files changed, 1270 insertions(+) diff --git a/.github/actions/check-container-upgrade/check-container-versions.py b/.github/actions/check-container-upgrade/check-container-versions.py new file mode 100755 index 000000000000..306c3e7a6e21 --- /dev/null +++ b/.github/actions/check-container-upgrade/check-container-versions.py @@ -0,0 +1,939 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +""" +Container Version Checker for Apache Camel Test Infrastructure + +This script scans all container.properties files in the test-infra modules and checks +for newer versions of container images in their respective registries. + +Supported Registries: + - Docker Hub (docker.io, registry-1.docker.io) + - Quay.io (quay.io) + - Google Container Registry (gcr.io, mirror.gcr.io) + - GitHub Container Registry (ghcr.io) + - IBM Container Registry (icr.io) + - Elastic Docker Registry (docker.elastic.co) + - NVIDIA Container Registry (nvcr.io) + - Microsoft Container Registry (mcr.microsoft.com) + - Weaviate Container Registry (cr.weaviate.io) + +Usage: + python3 check-container-versions.py [options] + +Options: + --verbose, -v Enable verbose output + --json Output results in JSON format + --check-prereleases Include pre-release versions in checks + --registry-timeout Registry API timeout in seconds (default: 30) + --help, -h Show this help message + +Dependencies: + pip install requests packaging colorama +""" + +import os +import re +import sys +import json +import argparse +import configparser +from pathlib import Path +from typing import Dict, List, Tuple, Optional, Any +from dataclasses import dataclass, asdict +from urllib.parse import urlparse +import requests +from packaging import version +from colorama import init, Fore, Style, Back +import time + +# Initialize colorama for cross-platform colored output +init(autoreset=True) + +@dataclass +class ContainerImage: + """Represents a container image with its registry, name, and version.""" + registry: str + namespace: str + name: str + current_version: str + property_name: str + file_path: str + + @property + def full_name(self) -> str: + """Returns the full image name without version.""" + if self.namespace: + return f"{self.registry}/{self.namespace}/{self.name}" + else: + return f"{self.registry}/{self.name}" + + @property + def full_image(self) -> str: + """Returns the complete image reference with version.""" + return f"{self.full_name}:{self.current_version}" + +@dataclass +class VersionCheckResult: + """Result of checking for newer versions of a container image.""" + image: ContainerImage + available_versions: List[str] + latest_version: Optional[str] + newer_versions: List[str] + is_latest: bool + error: Optional[str] = None + +class ContainerRegistryAPI: + """Base class for container registry API interactions.""" + + def __init__(self, timeout: int = 30): + self.timeout = timeout + self.session = requests.Session() + self.session.headers.update({ + 'User-Agent': 'Apache-Camel-Test-Infra-Version-Checker/1.0' + }) + + def get_available_versions(self, image: ContainerImage) -> List[str]: + """Get available versions for the given image.""" + raise NotImplementedError + + def normalize_registry_url(self, registry: str) -> str: + """Normalize registry URL for API calls.""" + if registry == "mirror.gcr.io": + return "gcr.io" + elif registry == "docker.io" or registry == "": + return "registry-1.docker.io" + return registry + +class DockerHubAPI(ContainerRegistryAPI): + """Docker Hub registry API implementation.""" + + def get_available_versions(self, image: ContainerImage) -> List[str]: + """Get available versions from Docker Hub.""" + try: + # Handle official images (no namespace) + if not image.namespace or image.namespace == "library": + repo_name = image.name + else: + repo_name = f"{image.namespace}/{image.name}" + + url = f"https://registry.hub.docker.com/v2/repositories/{repo_name}/tags/"; + + versions = [] + page = 1 + max_pages = 20 # Prevent infinite loops + + while page <= max_pages: + params = {'page': page, 'page_size': 100} + response = self.session.get(url, params=params, timeout=self.timeout) + + if response.status_code != 200: + if response.status_code == 404: + return [] # Repository not found + response.raise_for_status() + + data = response.json() + + for tag_info in data.get('results', []): + tag_name = tag_info.get('name', '') + if tag_name and tag_name != 'latest': + versions.append(tag_name) + + # Check if there are more pages + if not data.get('next'): + break + + page += 1 + + return versions + + except Exception as e: + raise Exception(f"Docker Hub API error: {str(e)}") + +class QuayAPI(ContainerRegistryAPI): + """Quay.io registry API implementation.""" + + def get_available_versions(self, image: ContainerImage) -> List[str]: + """Get available versions from Quay.io.""" + try: + repo_name = f"{image.namespace}/{image.name}" + url = f"https://quay.io/api/v1/repository/{repo_name}/tag/"; + + params = {'limit': 100, 'page': 1} + versions = [] + + while True: + response = self.session.get(url, params=params, timeout=self.timeout) + + if response.status_code != 200: + if response.status_code == 404: + return [] + response.raise_for_status() + + data = response.json() + + for tag_info in data.get('tags', []): + tag_name = tag_info.get('name', '') + if tag_name and tag_name != 'latest': + versions.append(tag_name) + + # Check if there are more pages + if not data.get('has_additional', False): + break + + params['page'] += 1 + if params['page'] > 20: # Prevent infinite loops + break + + return versions + + except Exception as e: + raise Exception(f"Quay.io API error: {str(e)}") + +class GCRAPI(ContainerRegistryAPI): + """Google Container Registry API implementation.""" + + def get_available_versions(self, image: ContainerImage) -> List[str]: + """Get available versions from GCR.""" + try: + # GCR uses Docker Registry v2 API + if image.namespace: + repo_name = f"{image.namespace}/{image.name}" + else: + repo_name = image.name + + url = f"https://{image.registry}/v2/{repo_name}/tags/list"; + + response = self.session.get(url, timeout=self.timeout) + + if response.status_code != 200: + if response.status_code == 404: + return [] + response.raise_for_status() + + data = response.json() + versions = data.get('tags', []) + + # Filter out 'latest' tag + return [v for v in versions if v != 'latest'] + + except Exception as e: + raise Exception(f"GCR API error: {str(e)}") + +class GHCRAPI(ContainerRegistryAPI): + """GitHub Container Registry API implementation.""" + + def get_available_versions(self, image: ContainerImage) -> List[str]: + """Get available versions from GHCR (GitHub Container Registry).""" + try: + # GHCR uses OCI Distribution API (Docker Registry v2 compatible) + if image.namespace: + repo_name = f"{image.namespace}/{image.name}" + else: + repo_name = image.name + + url = f"https://{image.registry}/v2/{repo_name}/tags/list"; + + # GHCR may require authentication for some repos, but we'll try anonymous first + response = self.session.get(url, timeout=self.timeout) + + if response.status_code != 200: + if response.status_code == 404: + return [] + response.raise_for_status() + + data = response.json() + versions = data.get('tags', []) + + # Filter out 'latest' tag + return [v for v in versions if v != 'latest'] + + except Exception as e: + raise Exception(f"GHCR API error: {str(e)}") + +class DockerV2RegistryAPI(ContainerRegistryAPI): + """Generic Docker Registry v2 API implementation for various registries.""" + + def __init__(self, timeout: int = 30, registry_name: str = "Docker Registry"): + super().__init__(timeout) + self.registry_name = registry_name + + def get_available_versions(self, image: ContainerImage) -> List[str]: + """Get available versions from a Docker Registry v2 compatible registry.""" + try: + # Build repository name + if image.namespace: + repo_name = f"{image.namespace}/{image.name}" + else: + repo_name = image.name + + url = f"https://{image.registry}/v2/{repo_name}/tags/list"; + + response = self.session.get(url, timeout=self.timeout) + + if response.status_code != 200: + if response.status_code == 404: + return [] + response.raise_for_status() + + data = response.json() + versions = data.get('tags', []) + + # Filter out 'latest' tag + return [v for v in versions if v != 'latest'] + + except Exception as e: + raise Exception(f"{self.registry_name} API error: {str(e)}") + +class ElasticRegistryAPI(ContainerRegistryAPI): + """Elastic Docker Registry API implementation.""" + + def get_available_versions(self, image: ContainerImage) -> List[str]: + """Get available versions from Elastic's Docker Registry.""" + try: + # Elastic uses a standard Docker Registry v2 API + if image.namespace: + repo_name = f"{image.namespace}/{image.name}" + else: + repo_name = image.name + + url = f"https://{image.registry}/v2/{repo_name}/tags/list"; + + response = self.session.get(url, timeout=self.timeout) + + if response.status_code != 200: + if response.status_code == 404: + return [] + response.raise_for_status() + + data = response.json() + versions = data.get('tags', []) + + # Filter out 'latest' tag + return [v for v in versions if v != 'latest'] + + except Exception as e: + raise Exception(f"Elastic Registry API error: {str(e)}") + +class NVIDIARegistryAPI(ContainerRegistryAPI): + """NVIDIA Container Registry API implementation.""" + + def get_available_versions(self, image: ContainerImage) -> List[str]: + """Get available versions from NVIDIA Container Registry.""" + try: + # NVIDIA NGC uses a Docker Registry v2 compatible API + if image.namespace: + repo_name = f"{image.namespace}/{image.name}" + else: + repo_name = image.name + + url = f"https://{image.registry}/v2/{repo_name}/tags/list"; + + response = self.session.get(url, timeout=self.timeout) + + if response.status_code != 200: + if response.status_code == 404: + return [] + response.raise_for_status() + + data = response.json() + versions = data.get('tags', []) + + # Filter out 'latest' tag + return [v for v in versions if v != 'latest'] + + except Exception as e: + raise Exception(f"NVIDIA Registry API error: {str(e)}") + +class MicrosoftRegistryAPI(ContainerRegistryAPI): + """Microsoft Container Registry API implementation.""" + + def get_available_versions(self, image: ContainerImage) -> List[str]: + """Get available versions from Microsoft Container Registry.""" + try: + # MCR uses Docker Registry v2 API + if image.namespace: + repo_name = f"{image.namespace}/{image.name}" + else: + repo_name = image.name + + url = f"https://{image.registry}/v2/{repo_name}/tags/list"; + + response = self.session.get(url, timeout=self.timeout) + + if response.status_code != 200: + if response.status_code == 404: + return [] + response.raise_for_status() + + data = response.json() + versions = data.get('tags', []) + + # Filter out 'latest' tag + return [v for v in versions if v != 'latest'] + + except Exception as e: + raise Exception(f"Microsoft Registry API error: {str(e)}") + +class ContainerVersionChecker: + """Main class for checking container versions.""" + + def __init__(self, + include_prereleases: bool = False, + registry_timeout: int = 30, + verbose: bool = False): + self.include_prereleases = include_prereleases + self.verbose = verbose + + # Initialize registry APIs + self.registry_apis = { + 'docker.io': DockerHubAPI(registry_timeout), + 'registry-1.docker.io': DockerHubAPI(registry_timeout), + 'quay.io': QuayAPI(registry_timeout), + 'gcr.io': GCRAPI(registry_timeout), + 'mirror.gcr.io': GCRAPI(registry_timeout), + 'ghcr.io': GHCRAPI(registry_timeout), + 'icr.io': DockerV2RegistryAPI(registry_timeout, "IBM Container Registry"), + 'docker.elastic.co': ElasticRegistryAPI(registry_timeout), + 'nvcr.io': NVIDIARegistryAPI(registry_timeout), + 'mcr.microsoft.com': MicrosoftRegistryAPI(registry_timeout), + 'cr.weaviate.io': DockerV2RegistryAPI(registry_timeout, "Weaviate Container Registry"), + } + + def parse_container_reference(self, container_ref: str) -> Tuple[str, str, str, str]: + """Parse container reference into registry, namespace, name, and version.""" + # Handle cases like: + # - postgres:17.5-alpine + # - mirror.gcr.io/postgres:17.5-alpine + # - quay.io/strimzi/kafka:latest-kafka-3.9.1 + # - mirror.gcr.io/confluentinc/cp-kafka:7.9.2 + + if '://' in container_ref: + # Remove protocol if present + container_ref = container_ref.split('://', 1)[1] + + # Split by ':' + parts = container_ref.rsplit(':', 1) + if len(parts) != 2: + raise ValueError(f"Invalid container reference: {container_ref}") + + image_part, version_part = parts + + # Split image part by '/' + image_components = image_part.split('/') + + if len(image_components) == 1: + # No registry specified, assume Docker Hub + registry = "docker.io" + namespace = "" + name = image_components[0] + elif len(image_components) == 2: + # Could be registry/image or namespace/image + if '.' in image_components[0] or image_components[0] in ['localhost', 'mirror']: + # Looks like a registry + registry = image_components[0] + namespace = "" + name = image_components[1] + else: + # Looks like namespace/image on Docker Hub + registry = "docker.io" + namespace = image_components[0] + name = image_components[1] + elif len(image_components) == 3: + # registry/namespace/image + registry = image_components[0] + namespace = image_components[1] + name = image_components[2] + else: + # More complex case, treat everything except last as registry/namespace + registry = image_components[0] + namespace = '/'.join(image_components[1:-1]) + name = image_components[-1] + + return registry, namespace, name, version_part + + def find_container_properties_files(self, base_path: str) -> List[str]: + """Find all container.properties files in the specified directory.""" + properties_files = [] + base_path = Path(base_path) + + if self.verbose: + print(f" Searching in: {base_path}") + + # Directories to exclude from search + exclude_dirs = {'target', 'build', '.git', 'node_modules', '__pycache__'} + + # Look for container.properties files anywhere in the directory tree + for properties_file in base_path.rglob("container.properties"): + # Skip files in excluded directories + if any(excluded in properties_file.parts for excluded in exclude_dirs): + if self.verbose: + print(f" Skipped (in excluded dir): {properties_file}") + continue + + properties_files.append(str(properties_file)) + if self.verbose: + print(f" Found: {properties_file}") + + # If no files found, also check for any .properties files that might contain container references + # If no container.properties files found, search for other patterns + if not properties_files: + if self.verbose: + print(" No container.properties files found, searching for alternatives...") + + # Check for other common container property file patterns + search_patterns = [ + "docker.properties", + "containers.properties", + "images.properties", + "testcontainers.properties" + ] + + for pattern in search_patterns: + for properties_file in base_path.rglob(pattern): + # Skip files in excluded directories + if any(excluded in properties_file.parts for excluded in exclude_dirs): + continue + properties_files.append(str(properties_file)) + if self.verbose: + print(f" Found by pattern {pattern}: {properties_file}") + + # If still nothing found, scan all .properties files for container references + if not properties_files: + if self.verbose: + print(" No specific container property files found, scanning all .properties files...") + + for properties_file in base_path.rglob("*.properties"): + # Skip files in excluded directories + if any(excluded in properties_file.parts for excluded in exclude_dirs): + continue + + # Skip very large files to avoid performance issues + try: + file_size = properties_file.stat().st_size + if file_size > 1024 * 1024: # Skip files larger than 1MB + continue + + with open(properties_file, 'r', encoding='utf-8') as f: + content = f.read() + # Look for patterns that suggest container definitions + if any(pattern in content.lower() for pattern in [ + '.container=', 'container.image=', '.image=', + 'docker.io', 'gcr.io', 'quay.io', 'ghcr.io', 'icr.io', + 'mcr.microsoft.com', 'nvcr.io', 'docker.elastic.co', + 'cr.weaviate.io', 'localhost:', 'registry' + ]): + properties_files.append(str(properties_file)) + if self.verbose: + print(f" Found container references in: {properties_file}") + except (UnicodeDecodeError, IOError, OSError): + # Skip files that can't be read + continue + + return sorted(properties_files) + + def parse_properties_file(self, file_path: str) -> List[ContainerImage]: + """Parse a container.properties file and extract container images.""" + images = [] + + try: + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + + # Parse properties manually to handle comments + for line_num, line in enumerate(content.splitlines(), 1): + line = line.strip() + + # Skip comments and empty lines + if not line or line.startswith('#') or line.startswith('##'): + continue + + # Look for property=value format + if '=' in line: + key, value = line.split('=', 1) + key = key.strip() + value = value.strip() + + # Skip if value looks like a property reference + if value.startswith('${') or not value: + continue + + try: + registry, namespace, name, current_version = self.parse_container_reference(value) + + image = ContainerImage( + registry=registry, + namespace=namespace, + name=name, + current_version=current_version, + property_name=key, + file_path=file_path + ) + images.append(image) + + if self.verbose: + print(f" Found: {key} = {value}") + + except ValueError as e: + if self.verbose: + print(f" Warning: Could not parse {key}={value}: {e}") + continue + + except Exception as e: + print(f"Error reading {file_path}: {e}") + return [] + + return images + + def check_image_versions(self, image: ContainerImage) -> VersionCheckResult: + """Check for newer versions of a container image.""" + try: + # Get the appropriate API for the registry + registry_key = image.registry.lower() + + # Handle registry aliases and special cases + if registry_key == "mirror.gcr.io": + registry_key = "gcr.io" + elif registry_key in ["", "docker.io"]: + registry_key = "docker.io" + elif registry_key == "registry-1.docker.io": + registry_key = "docker.io" + + api = self.registry_apis.get(registry_key) + if not api: + return VersionCheckResult( + image=image, + available_versions=[], + latest_version=None, + newer_versions=[], + is_latest=False, + error=f"Unsupported registry: {image.registry}" + ) + + if self.verbose: + print(f" Checking {image.full_image}...") + + # Get available versions + available_versions = api.get_available_versions(image) + + if not available_versions: + return VersionCheckResult( + image=image, + available_versions=[], + latest_version=None, + newer_versions=[], + is_latest=True, + error="No versions found in registry" + ) + + # Sort versions + def version_sort_key(v): + try: + return version.parse(v) + except version.InvalidVersion: + # For non-semantic versions, sort lexicographically + return v + + try: + sorted_versions = sorted(available_versions, key=version_sort_key, reverse=True) + except: + # If sorting fails, use lexicographic sort + sorted_versions = sorted(available_versions, reverse=True) + + # Find newer versions + newer_versions = [] + current_ver = image.current_version + + for ver in sorted_versions: + try: + if version.parse(ver) > version.parse(current_ver): + if self.include_prereleases or not version.parse(ver).is_prerelease: + newer_versions.append(ver) + except version.InvalidVersion: + # For non-semantic versions, do string comparison + if ver > current_ver: + newer_versions.append(ver) + + latest_version = sorted_versions[0] if sorted_versions else None + is_latest = current_ver == latest_version or len(newer_versions) == 0 + + return VersionCheckResult( + image=image, + available_versions=sorted_versions[:10], # Limit to top 10 + latest_version=latest_version, + newer_versions=newer_versions[:5], # Limit to top 5 newer + is_latest=is_latest + ) + + except Exception as e: + return VersionCheckResult( + image=image, + available_versions=[], + latest_version=None, + newer_versions=[], + is_latest=False, + error=str(e) + ) + + def run_check(self, scan_path: str) -> List[VersionCheckResult]: + """Run the version check on all container.properties files.""" + print(f"š Scanning for container.properties files in {scan_path}...") + + properties_files = self.find_container_properties_files(scan_path) + + if not properties_files: + print("ā No container.properties files found!") + return [] + + print(f"š Found {len(properties_files)} container.properties files") + + all_images = [] + + # Parse all properties files + for file_path in properties_files: + if self.verbose: + print(f"\nš Parsing {file_path}...") + + images = self.parse_properties_file(file_path) + all_images.extend(images) + + if not all_images: + print("ā No container images found in properties files!") + return [] + + print(f"š³ Found {len(all_images)} container images") + print("š Checking for newer versions...") + + # Check versions for all images + results = [] + for i, image in enumerate(all_images, 1): + print(f" [{i}/{len(all_images)}] {image.full_image}") + result = self.check_image_versions(image) + results.append(result) + time.sleep(0.1) # Be nice to APIs + + return results + +def print_registry_summary(results: List[VersionCheckResult]): + """Print a summary of registries used.""" + registry_counts = {} + for result in results: + registry = result.image.registry + if registry in registry_counts: + registry_counts[registry] += 1 + else: + registry_counts[registry] = 1 + + if registry_counts: + print(f"\n{Style.BRIGHT}š REGISTRY USAGE SUMMARY{Style.RESET_ALL}") + print("-" * 35) + for registry, count in sorted(registry_counts.items(), key=lambda x: x[1], reverse=True): + print(f" {Fore.CYAN}{registry:<25}{Style.RESET_ALL} {count:>3} images") + +def print_results(results: List[VersionCheckResult], verbose: bool = False): + """Print the results in a human-readable format.""" + + # Separate results into categories + outdated = [r for r in results if not r.is_latest and not r.error and r.newer_versions] + up_to_date = [r for r in results if r.is_latest and not r.error] + errors = [r for r in results if r.error] + + print(f"\n{Style.BRIGHT}š VERSION CHECK SUMMARY{Style.RESET_ALL}") + print("=" * 50) + + print(f"Total images checked: {len(results)}") + print(f"{Fore.GREEN}ā Up to date: {len(up_to_date)}{Style.RESET_ALL}") + print(f"{Fore.YELLOW}ā ļø Outdated: {len(outdated)}{Style.RESET_ALL}") + print(f"{Fore.RED}ā Errors: {len(errors)}{Style.RESET_ALL}") + + if outdated: + print(f"\n{Style.BRIGHT}{Fore.YELLOW}š¦ OUTDATED IMAGES{Style.RESET_ALL}") + print("-" * 30) + + for result in sorted(outdated, key=lambda x: len(x.newer_versions), reverse=True): + image = result.image + print(f"\n{Fore.CYAN}{image.property_name}{Style.RESET_ALL}") + print(f" File: {os.path.relpath(image.file_path)}") + print(f" Current: {Fore.YELLOW}{image.current_version}{Style.RESET_ALL}") + print(f" Latest: {Fore.GREEN}{result.latest_version}{Style.RESET_ALL}") + + if len(result.newer_versions) > 1: + newer_display = result.newer_versions[:3] + if len(result.newer_versions) > 3: + newer_display.append(f"... (+{len(result.newer_versions) - 3} more)") + print(f" Newer: {Fore.GREEN}{', '.join(newer_display)}{Style.RESET_ALL}") + + if errors: + print(f"\n{Style.BRIGHT}{Fore.RED}ā ERRORS{Style.RESET_ALL}") + print("-" * 15) + + for result in errors: + image = result.image + print(f"\n{Fore.CYAN}{image.property_name}{Style.RESET_ALL}") + print(f" Image: {image.full_image}") + print(f" Error: {Fore.RED}{result.error}{Style.RESET_ALL}") + + if verbose and up_to_date: + print(f"\n{Style.BRIGHT}{Fore.GREEN}ā UP TO DATE IMAGES{Style.RESET_ALL}") + print("-" * 25) + + for result in up_to_date: + image = result.image + print(f" {image.property_name}: {image.current_version}") + +def output_json(results: List[VersionCheckResult]) -> str: + """Output results in JSON format.""" + json_results = [] + + for result in results: + json_result = { + 'image': asdict(result.image), + 'available_versions': result.available_versions, + 'latest_version': result.latest_version, + 'newer_versions': result.newer_versions, + 'is_latest': result.is_latest, + 'error': result.error + } + json_results.append(json_result) + + return json.dumps(json_results, indent=2) + +def main(): + parser = argparse.ArgumentParser( + description='Check for newer versions of container images in Apache Camel test-infra', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Basic usage + python3 check-container-versions.py + + # Verbose output + python3 check-container-versions.py --verbose + + # JSON output for automation + python3 check-container-versions.py --json + + # Include pre-release versions + python3 check-container-versions.py --check-prereleases + """ + ) + + parser.add_argument( + '--verbose', '-v', + action='store_true', + help='Enable verbose output' + ) + + parser.add_argument( + '--json', + action='store_true', + help='Output results in JSON format' + ) + + parser.add_argument( + '--check-prereleases', + action='store_true', + help='Include pre-release versions in checks' + ) + + parser.add_argument( + '--registry-timeout', + type=int, + default=30, + help='Registry API timeout in seconds (default: 30)' + ) + + parser.add_argument( + '--test-infra-path', + type=str, + default=None, + help='Path to test-infra directory or any directory containing container.properties files (default: current directory)' + ) + + parser.add_argument( + '--scan-path', + type=str, + default=None, + help='Alias for --test-infra-path (scan any directory for container.properties files)' + ) + + args = parser.parse_args() + + # Determine scan path (support both --test-infra-path and --scan-path) + scan_path = args.test_infra_path or args.scan_path + + if scan_path: + # Use specified path (convert to absolute path) + scan_path = os.path.abspath(os.path.expanduser(scan_path)) + if not os.path.exists(scan_path): + print(f"ā Specified path does not exist: {scan_path}") + sys.exit(1) + if not os.path.isdir(scan_path): + print(f"ā Specified path is not a directory: {scan_path}") + sys.exit(1) + else: + # Use current working directory + scan_path = os.getcwd() + + # Convert to absolute path for consistent behavior + scan_path = os.path.abspath(scan_path) + + if not args.json: + print(f"{Style.BRIGHT}š³ Apache Camel Test Infrastructure Container Version Checker{Style.RESET_ALL}") + print("=" * 60) + + # Create checker and run + checker = ContainerVersionChecker( + include_prereleases=args.check_prereleases, + registry_timeout=args.registry_timeout, + verbose=args.verbose + ) + + try: + results = checker.run_check(scan_path) + + if args.json: + print(output_json(results)) + else: + print_results(results, args.verbose) + + # Show registry usage summary if verbose + if args.verbose: + print_registry_summary(results) + + # Exit with non-zero if there are outdated images + outdated_count = len([r for r in results if not r.is_latest and not r.error and r.newer_versions]) + if outdated_count > 0: + print(f"\nš” Found {outdated_count} outdated images. Consider updating!") + sys.exit(1) + else: + print(f"\nš All images are up to date!") + + except KeyboardInterrupt: + print(f"\n{Fore.YELLOW}ā ļø Interrupted by user{Style.RESET_ALL}") + sys.exit(130) + except Exception as e: + if args.json: + print(json.dumps({"error": str(e)}, indent=2)) + else: + print(f"{Fore.RED}ā Error: {e}{Style.RESET_ALL}") + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.github/workflows/check-container-versions.yml b/.github/workflows/check-container-versions.yml new file mode 100644 index 000000000000..d5e977929cbd --- /dev/null +++ b/.github/workflows/check-container-versions.yml @@ -0,0 +1,331 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Container Version Upgrade + +on: + schedule: + # Run every Monday at 6:00 AM UTC + - cron: '0 6 * * MON' + workflow_dispatch: + inputs: + scan_path: + description: 'Path to scan for container.properties files' + required: false + default: 'test-infra' + check_prereleases: + description: 'Include pre-release versions' + required: false + type: boolean + default: false + +jobs: + upgrade-container-versions: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install requests packaging colorama + + - name: Check container versions + id: version_check + run: | + SCAN_PATH="${{ github.event.inputs.scan_path || 'test-infra' }}" + PRERELEASE_FLAG="" + + if [[ "${{ github.event.inputs.check_prereleases }}" == "true" ]]; then + PRERELEASE_FLAG="--check-prereleases" + fi + + echo "Scanning path: $SCAN_PATH" + + # Run the version checker and capture output + if python3 ./github/actions/check-container-upgrade/check-container-versions.py --scan-path "$SCAN_PATH" $PRERELEASE_FLAG --json > versions.json; then + echo "check_passed=true" >> $GITHUB_OUTPUT + echo "outdated_count=0" >> $GITHUB_OUTPUT + else + echo "check_passed=false" >> $GITHUB_OUTPUT + + # Count outdated images + OUTDATED_COUNT=$(jq '[.[] | select(.is_latest == false and .newer_versions != null and (.newer_versions | length) > 0)] | length' versions.json) + echo "outdated_count=$OUTDATED_COUNT" >> $GITHUB_OUTPUT + fi + + - name: Update container.properties files + if: steps.version_check.outputs.check_passed == 'false' + id: update_files + run: | + # Create a script to update the properties files + cat > update_versions.py << 'SCRIPT_EOF' + import json + import re + import sys + + # Load the version check results + with open('versions.json', 'r') as f: + results = json.load(f) + + updated_files = [] + updates = [] + + for result in results: + # Skip if already latest or has errors + if result.get('is_latest') or result.get('error'): + continue + + newer_versions = result.get('newer_versions', []) + if not newer_versions: + continue + + image = result['image'] + file_path = image['file_path'] + property_name = image['property_name'] + current_version = image['current_version'] + latest_version = result['latest_version'] + + # Read the properties file + try: + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + + # Build the full image reference for find/replace + registry = image['registry'] + namespace = image['namespace'] + name = image['name'] + + # Construct the image path + if namespace: + image_path = f"{registry}/{namespace}/{name}" if registry not in ['docker.io', ''] else f"{namespace}/{name}" + else: + image_path = f"{registry}/{name}" if registry not in ['docker.io', ''] else name + + # Create the old and new references + old_ref = f"{image_path}:{current_version}" + new_ref = f"{image_path}:{latest_version}" + + # Also handle case where registry might be implicit + if registry in ['docker.io', '']: + # Try both with and without explicit registry + old_pattern = f"(docker\\.io/)?{re.escape(image_path)}:{re.escape(current_version)}" + new_content = re.sub(old_pattern, new_ref, content) + else: + # Direct replacement + new_content = content.replace(old_ref, new_ref) + + if new_content != content: + # Write the updated content + with open(file_path, 'w', encoding='utf-8') as f: + f.write(new_content) + + updated_files.append(file_path) + updates.append({ + 'property': property_name, + 'file': file_path, + 'old_version': current_version, + 'new_version': latest_version, + 'image_name': image_path + }) + + print(f"ā Updated {property_name}: {current_version} ā {latest_version}") + else: + print(f"ā ļø Could not update {property_name} in {file_path}") + + except Exception as e: + print(f"ā Error updating {file_path}: {e}") + continue + + # Save the updates for PR body + with open('updates.json', 'w') as f: + json.dump(updates, f, indent=2) + + print(f"\nTotal files updated: {len(updated_files)}") + sys.exit(0 if updated_files else 1) + SCRIPT_EOF + + # Run the update script + if python3 update_versions.py; then + echo "updates_made=true" >> $GITHUB_OUTPUT + else + echo "updates_made=false" >> $GITHUB_OUTPUT + fi + + - name: Generate PR description + if: steps.update_files.outputs.updates_made == 'true' + id: pr_description + run: | + # Generate PR body + cat > pr_body.md << 'EOF' + This PR updates outdated container images in test-infra to their latest versions. + + ## Updated Images + + EOF + + # Add details from updates.json + python3 << 'PYTHON_EOF' + import json + + with open('updates.json', 'r') as f: + updates = json.load(f) + + for update in updates: + print(f"### {update['property']}") + print(f"- **Image**: `{update['image_name']}`") + print(f"- **File**: `{update['file']}`") + print(f"- **Old version**: `{update['old_version']}`") + print(f"- **New version**: `{update['new_version']}`") + print() + PYTHON_EOF >> pr_body.md + + cat >> pr_body.md << 'EOF' + + ## Verification + + Please verify: + - [ ] All container image versions are compatible with existing tests + - [ ] No breaking changes in the updated versions + - [ ] Tests pass with the new versions + + --- + + This PR was automatically created by the [Container Version Upgrade workflow](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}). + EOF + + # Set output for PR body + { + echo 'pr_body<<PR_BODY_EOF' + cat pr_body.md + echo 'PR_BODY_EOF' + } >> $GITHUB_OUTPUT + + # Generate commit message + UPDATE_COUNT=$(jq 'length' updates.json) + echo "commit_message=chore(test-infra): upgrade container images ($UPDATE_COUNT images)" >> $GITHUB_OUTPUT + + - name: Create Pull Request + if: steps.update_files.outputs.updates_made == 'true' + id: create_pr + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: | + ${{ steps.pr_description.outputs.commit_message }} + + Updated container images to latest versions: + ${{ steps.pr_description.outputs.pr_body }} + + š¤ Generated with Claude Code + branch: automated/container-version-upgrade-${{ github.run_number }} + delete-branch: true + title: 'chore(test-infra): Upgrade container images to latest versions' + body: ${{ steps.pr_description.outputs.pr_body }} + labels: | + dependencies + container-images + automated + draft: false + + - name: Upload results artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: container-version-check-results + path: | + versions.json + updates.json + pr_body.md + retention-days: 30 + + - name: Job summary + if: always() + run: | + if [[ -f pr_body.md ]]; then + cat pr_body.md >> $GITHUB_STEP_SUMMARY + if [[ -n "${{ steps.create_pr.outputs.pull-request-number }}" ]]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "š¬ **Pull Request Created**: #${{ steps.create_pr.outputs.pull-request-number }}" >> $GITHUB_STEP_SUMMARY + echo "š **URL**: ${{ steps.create_pr.outputs.pull-request-url }}" >> $GITHUB_STEP_SUMMARY + fi + else + echo "ā All container images are up to date!" >> $GITHUB_STEP_SUMMARY + fi + + - name: Comment on PR with details + if: steps.create_pr.outputs.pull-request-number != '' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + + // Read the versions.json for additional context + const versions = JSON.parse(fs.readFileSync('versions.json', 'utf8')); + const updates = JSON.parse(fs.readFileSync('updates.json', 'utf8')); + + let comment = '## Container Update Details\n\n'; + comment += `Total images checked: ${versions.length}\n`; + comment += `Images updated: ${updates.length}\n\n`; + + comment += '### Registry Distribution\n\n'; + const registries = {}; + updates.forEach(u => { + const registry = u.image_name.split('/')[0]; + registries[registry] = (registries[registry] || 0) + 1; + }); + + for (const [registry, count] of Object.entries(registries)) { + comment += `- **${registry}**: ${count} image(s)\n`; + } + + comment += '\n### Testing Recommendations\n\n'; + comment += 'Run the following to verify the updates:\n'; + comment += '```bash\n'; + comment += 'mvn clean verify -pl '; + + // Extract unique modules from file paths + const modules = new Set(); + updates.forEach(u => { + const match = u.file.match(/test-infra\/(camel-test-infra-[^\/]+)/); + if (match) { + modules.add(match[1]); + } + }); + + comment += Array.from(modules).join(','); + comment += '\n```\n'; + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ${{ steps.create_pr.outputs.pull-request-number }}, + body: comment + });
