potiuk commented on a change in pull request #13767:
URL: https://github.com/apache/airflow/pull/13767#discussion_r561742545
##########
File path: dev/provider_packages/prepare_provider_packages.py
##########
@@ -1225,184 +1195,272 @@ def
get_provider_info_from_provider_yaml(provider_package_id: str) -> Dict[str,
return provider_info
-def update_generated_files_for_package(
+def get_backport_current_changes_table(previous_release_commit_ref: str,
source_provider_package_path: str):
+ git_cmd = get_git_log_command(previous_release_commit_ref)
+ try:
+ changes = subprocess.check_output(git_cmd,
cwd=source_provider_package_path, universal_newlines=True)
+ changes_table = convert_git_changes_to_table(
+ previous_release_commit_ref, changes,
base_url="https://github.com/apache/airflow/commit/";
+ )
+ except subprocess.CalledProcessError:
+ changes_table = ''
+ return changes_table
+
+
+def get_version_tag(version: str, provider_package_id: str, version_suffix:
str):
+ return
f"providers-{provider_package_id.replace('.','-')}/{version}{version_suffix}"
+
+
+def print_changes_table(changes_table):
+ syntax = Syntax(changes_table, "rst", theme="light")
+ console = Console(width=200)
+ console.print(syntax)
+
+
+def get_all_changes_for_regular_packages(
+ versions: List[str],
provider_package_id: str,
- current_release_version: str,
+ source_provider_package_path: str,
version_suffix: str,
- imported_classes: List[str],
- backport_packages: bool,
- update_release_notes: bool,
- update_setup: bool,
-) -> Tuple[int, int]:
- """
- Updates release notes (BACKPORT_PROVIDER_README.md/README.md) for the
package.
- Returns Tuple of total number of entities and badly named entities.
+) -> Tuple[bool, str]:
+ current_version = versions[0]
+ current_tag = get_version_tag(current_version, provider_package_id,
version_suffix)
+ print(f"Checking if tag `{current_tag}` exists.")
+ if not subprocess.call(
+ get_git_tag_check_command(current_tag),
+ cwd=source_provider_package_path,
+ stderr=subprocess.DEVNULL,
+ ):
+ print(f"The tag {current_tag} exists.")
+ # The tag already exists
+ changes = subprocess.check_output(
+ get_git_log_command(current_tag, 'HEAD'),
+ cwd=source_provider_package_path,
+ universal_newlines=True,
+ )
+ if len(changes) > 0:
+ print(
+ f"[yellow]The provider {provider_package_id} has changes"
+ f" since last release but version is not updated[/]"
+ )
+ print()
+ print(
+ f'Please update version in
`airflow/providers/{provider_package_id.replace("-","/")}/'
+ 'provider.yaml` to prepare release.\n'
+ )
+ changes_table = convert_git_changes_to_table(
+ "UNKNOWN", changes,
base_url="https://github.com/apache/airflow/commit/";, markdown=False
+ )
+ print_changes_table(changes_table)
+ return False, changes_table
+ else:
+ print(f"No changes for {provider_package_id}")
+ return False, ""
+ print("The tag does not exist. ")
+ if len(versions) == 1:
+ print("This provider has never been released!\n")
+ else:
+ print(f"The {provider_package_id} is ready to be released!\n")
+ current_version = 'HEAD'
+ changes_table = ''
+ print_version = versions[0]
+ for version in versions[1:]:
+ version_tag = get_version_tag(version, provider_package_id,
version_suffix)
+ changes = subprocess.check_output(
+ get_git_log_command(version_tag, current_version),
+ cwd=source_provider_package_path,
+ universal_newlines=True,
+ )
+ changes_table += convert_git_changes_to_table(
+ print_version, changes,
base_url="https://github.com/apache/airflow/commit/";, markdown=False
+ )
+ current_version = version
+ print_version = version
+ print_changes_table(changes_table)
+ return True, changes_table
+
+
+def get_provider_details(provider_package_id: str) -> ProviderPackageDetails:
+ return ProviderPackageDetails(
+ provider_package_id=provider_package_id,
+ full_package_name=f"airflow.providers.{provider_package_id}",
+
source_provider_package_path=get_source_package_path(provider_package_id),
+
documentation_provider_package_path=get_documentation_package_path(provider_package_id),
+ )
- :param provider_package_id: id of the package
- :param current_release_version: release version:
- :param version_suffix: version suffix corresponding to the version in the
code
- :param imported_classes - entities that have been imported from providers
- :param backport_packages: whether to prepare regular (False) or backport
(True) packages
- :param update_release_notes: whether to update release notes
- :param update_setup: whether to update setup files
- :return: Tuple of total/bad number of entities
- """
- verify_provider_package(provider_package_id)
- full_package_name = f"airflow.providers.{provider_package_id}"
- source_provider_package_path = get_source_package_path(provider_package_id)
- entity_summaries = get_package_class_summary(full_package_name,
imported_classes)
- past_releases = get_all_releases(
- provider_package_path=source_provider_package_path,
backport_packages=backport_packages
+def get_provider_jinja_context(
+ provider_details: ProviderPackageDetails,
+ current_release_version: str,
+ version_suffix: str,
+ backport_packages: bool,
+):
+ verify_provider_package(provider_details.provider_package_id)
+ cross_providers_dependencies = get_cross_provider_dependent_packages(
+ provider_package_id=provider_details.provider_package_id
)
- current_release_version, previous_release = check_if_release_version_ok(
- past_releases, current_release_version, backport_packages
+ release_version_no_leading_zeros =
strip_leading_zeros(current_release_version)
+ pip_requirements_table = convert_pip_requirements_to_table(
+ PROVIDERS_REQUIREMENTS[provider_details.provider_package_id]
)
- cross_providers_dependencies = get_cross_provider_dependent_packages(
- provider_package_id=provider_package_id
+ pip_requirements_table_rst = convert_pip_requirements_to_table(
+ PROVIDERS_REQUIREMENTS[provider_details.provider_package_id],
markdown=False
)
- previous_release = get_previous_release_info(
- previous_release_version=previous_release,
- past_releases=past_releases,
- current_release_version=current_release_version,
+ cross_providers_dependencies_table =
convert_cross_package_dependencies_to_table(
+ cross_providers_dependencies,
+ backport_packages=backport_packages,
)
- release_version_no_leading_zeros = (
- strip_leading_zeros_in_calver(current_release_version)
- if backport_packages
- else current_release_version
+ cross_providers_dependencies_table_rst =
convert_cross_package_dependencies_to_table(
+ cross_providers_dependencies, backport_packages=backport_packages,
markdown=False
)
context: Dict[str, Any] = {
"ENTITY_TYPES": list(EntityType),
"README_FILE": "BACKPORT_PROVIDER_README.md" if backport_packages else
"README.md",
- "PROVIDER_PACKAGE_ID": provider_package_id,
- "PACKAGE_PIP_NAME": get_pip_package_name(provider_package_id,
backport_packages),
- "FULL_PACKAGE_NAME": full_package_name,
- "PROVIDER_PATH": full_package_name.replace(".", "/"),
+ "PROVIDER_PACKAGE_ID": provider_details.provider_package_id,
+ "PACKAGE_PIP_NAME": get_pip_package_name(
+ provider_details.provider_package_id,
backport_packages=backport_packages
+ ),
+ "FULL_PACKAGE_NAME": provider_details.full_package_name,
+ "PROVIDER_PATH": provider_details.full_package_name.replace(".", "/"),
"RELEASE": current_release_version,
"RELEASE_NO_LEADING_ZEROS": release_version_no_leading_zeros,
"VERSION_SUFFIX": version_suffix,
- "ADDITIONAL_INFO":
get_additional_package_info(provider_package_path=source_provider_package_path),
+ "ADDITIONAL_INFO": get_additional_package_info(
+ provider_package_path=provider_details.source_provider_package_path
+ ),
+ "CHANGELOG": get_changelog_for_package(
+ provider_package_path=provider_details.source_provider_package_path
+ ),
"CROSS_PROVIDERS_DEPENDENCIES": cross_providers_dependencies,
- "PIP_REQUIREMENTS": PROVIDERS_REQUIREMENTS[provider_package_id],
- "PROVIDER_TYPE": "Backport provider" if BACKPORT_PACKAGES else
"Provider",
- "PROVIDERS_FOLDER": "backport-providers" if BACKPORT_PACKAGES else
"providers",
+ "PIP_REQUIREMENTS":
PROVIDERS_REQUIREMENTS[provider_details.provider_package_id],
+ "PROVIDER_TYPE": "Backport provider" if backport_packages else
"Provider",
+ "PROVIDERS_FOLDER": "backport-providers" if backport_packages else
"providers",
"INSTALL_REQUIREMENTS": get_install_requirements(
- provider_package_id=provider_package_id,
backport_packages=backport_packages
+ provider_package_id=provider_details.provider_package_id,
backport_packages=backport_packages
),
"SETUP_REQUIREMENTS": get_setup_requirements(),
"EXTRAS_REQUIREMENTS": get_package_extras(
- provider_package_id=provider_package_id,
backport_packages=backport_packages
+ provider_package_id=provider_details.provider_package_id,
backport_packages=backport_packages
),
- "PROVIDER_INFO":
get_provider_info_from_provider_yaml(provider_package_id),
+ "CROSS_PROVIDERS_DEPENDENCIES_TABLE":
cross_providers_dependencies_table,
+ "CROSS_PROVIDERS_DEPENDENCIES_TABLE_RST":
cross_providers_dependencies_table_rst,
+ "PIP_REQUIREMENTS_TABLE": pip_requirements_table,
+ "PIP_REQUIREMENTS_TABLE_RST": pip_requirements_table_rst,
}
- if update_release_notes:
- git_cmd = get_git_command(previous_release)
- try:
- changes = subprocess.check_output(
- git_cmd, cwd=source_provider_package_path,
universal_newlines=True
- )
- changes_table = convert_git_changes_to_table(
- changes, base_url="https://github.com/apache/airflow/commit/";
- )
- except subprocess.CalledProcessError:
- # TODO(potiuk) fix me for both backport/provider package check
- changes_table = ''
- context["CURRENT_CHANGES_TABLE"] = changes_table
- pip_requirements_table = convert_pip_requirements_to_table(
- PROVIDERS_REQUIREMENTS[provider_package_id]
- )
- cross_providers_dependencies_table =
convert_cross_package_dependencies_to_table(
- cross_providers_dependencies,
- backport_packages=backport_packages,
- )
- context["CROSS_PROVIDERS_DEPENDENCIES_TABLE"] =
cross_providers_dependencies_table
- context["PIP_REQUIREMENTS_TABLE"] = pip_requirements_table
+ return context
- total, bad = check_if_classes_are_properly_named(entity_summaries)
+
+def prepare_readme_file(
+ context,
+):
+ readme_content = LICENCE
+ readme_template_name = PROVIDER_TEMPLATE_PREFIX + "README"
+ readme_content += render_template(template_name=readme_template_name,
context=context, extension='.md')
+ readme_file_path = os.path.join(TARGET_PROVIDER_PACKAGES_PATH, "README.md")
+ with open(readme_file_path, "wt") as readme_file:
+ readme_file.write(readme_content)
+
+
+def update_generated_files_for_regular_package(
+ provider_package_id: str,
+ version_suffix: str,
+ update_release_notes: bool,
+ update_setup: bool,
+) -> None:
+ """
+ Updates generated files (readme, changes and/or
setup.cfg/setup.py/manifest.in/provider_info)
+
+ :param provider_package_id: id of the package
+ :param version_suffix: version suffix corresponding to the version in the
code
+ :param update_release_notes: whether to update release notes
+ :param update_setup: whether to update setup files
+ """
+ verify_provider_package(provider_package_id)
+ provider_details = get_provider_details(provider_package_id)
+ provider_info = get_provider_info_from_provider_yaml(provider_package_id)
+ versions: List[str] = provider_info['versions']
+ current_release_version = versions[0]
+ # previous_release = versions[1] if len(versions) > 1 else None
+ # previous_release_commit_ref =
f"providers-{provider_package_id.replace('.','-')}/{previous_release}"
Review comment:
Yep
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
