dstandish commented on a change in pull request #19238:
URL: https://github.com/apache/airflow/pull/19238#discussion_r737635499
##########
File path: dev/check_files.py
##########
@@ -86,66 +83,80 @@ def create_docker(txt: str):
)
-def check_all_present(prefix: str, files: List[str]):
- all_present = True
- for ext in [ASC, SHA, NORM]:
- if any(re.match(ext, f) for f in files):
- print(f" - {prefix} {ext.pattern}: [green]OK[/green]")
- else:
- print(f" - {prefix} {ext.pattern}: [red]MISSING[/red]")
- all_present = False
- return all_present
+def check_providers(files: List[str], version: str):
+ print(f"Checking providers for version {version}:\n")
+ version = strip_rc_suffix(version)
+ missing_list = []
+ for p in get_packages():
+ print(p)
+ expected_files = expand_name_variations(
+ [
+ f"{p}-{version}.tar.gz",
+ f"{p.replace('-', '_')}-{version}-py3-none-any.whl",
+ ]
+ )
+ missing_list.extend(check_all_files(expected_files=expected_files,
actual_files=files))
-def filter_files(files: List[str], prefix: str):
- return [f for f in files if f.startswith(prefix)]
+ return missing_list
-def check_providers(files: List[str], version: str):
- name_tpl = "apache_airflow_providers_{}-{}"
- pip_packages = []
- for p in get_packages():
- print(p)
+def strip_rc_suffix(version):
+ return re.sub(r'rc\d+$', '', version)
- name = name_tpl.format(p.replace(".", "_"), version)
- # Check sources
- check_all_present("sources", filter_files(files, name))
- # Check wheels
- name = name.replace("_", "-")
- if check_all_present("wheel", filter_files(files, name)):
- pip_packages.append(f"{name.rpartition('-')[0]}=={version}")
+def print_status(file, is_found: bool):
+ color, status = ('green', 'OK') if is_found else ('red', 'MISSING')
+ print(f" - {file}: [{color}]{status}[/{color}]")
- return pip_packages
+def check_all_files(actual_files, expected_files):
+ missing_list = []
+ for file in expected_files:
+ is_found = file in actual_files
+ if not is_found:
+ missing_list.append(file)
+ print_status(file=file, is_found=is_found)
+ return missing_list
-def check_release(files: List[str], version: str):
- print(f"apache_airflow-{version}")
- # Check bin
- name = f"apache-airflow-{version}-bin"
- check_all_present("binaries", filter_files(files, name))
+def check_release(files: List[str], version: str):
+ print(f"Checking airflow release for version {version}:\n")
+ version = strip_rc_suffix(version)
+
+ expected_files = expand_name_variations(
+ [
+ f"apache-airflow-{version}.tar.gz",
+ f"apache-airflow-{version}-source.tar.gz",
+ f"apache_airflow-{version}-py3-none-any.whl",
+ ]
+ )
+ return check_all_files(expected_files=expected_files, actual_files=files)
- # Check sources
- name = f"apache-airflow-{version}-source"
- check_all_present("sources", filter_files(files, name))
- # Check wheels
- name = f"apache_airflow-{version}-py"
- check_all_present("wheel", filter_files(files, name))
+def expand_name_variations(files):
+ return list(sorted(base + suffix for base, suffix in product(files, ['',
'.asc', '.sha512'])))
def check_upgrade_check(files: List[str], version: str):
- print(f"apache_airflow-upgrade-check-{version}")
+ print(f"Checking upgrade_check for version {version}:\n")
+ version = strip_rc_suffix(version)
+
+ expected_files = expand_name_variations(
+ [
+ f"apache-airflow-upgrade-check-{version}-bin.tar.gz",
+ f"apache-airflow-upgrade-check-{version}-source.tar.gz",
+ f"apache_airflow_upgrade_check-{version}-py2.py3-none-any.whl",
+ ]
+ )
+ return check_all_files(expected_files=expected_files, actual_files=files)
- name = f"apache-airflow-upgrade-check-{version}-bin"
- check_all_present("binaries", filter_files(files, name))
- name = f"apache-airflow-upgrade-check-{version}-source"
- check_all_present("sources", filter_files(files, name))
+def warn_of_missing_files(files):
+ print(f"[red]Check failed. Here are the files we expected but did not
find:[/red]\n")
Review comment:
sorry fixed before saw you did the work
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
