mik-laj commented on a change in pull request #8807:
URL: https://github.com/apache/airflow/pull/8807#discussion_r422774281
##########
File path: backport_packages/setup_backport_packages.py
##########
@@ -422,13 +371,376 @@ def usage():
print(text)
print()
print("You can see all packages configured by specifying
list-backport-packages as first argument")
+ print("You can generate release notes by specifying:"
+ " update-package-release-notes YYYY.MM.DD [PACKAGES]")
+
+
+def is_imported_google_base_hook(name: str) -> bool:
+ if name.endswith("GoogleBaseHook") and name != \
+ "airflow.providers.google.common.hooks.base_google.GoogleBaseHook":
+ return True
+ return False
+
+
+# return list of tuples (objclass, name) containing all subclasses in package
specified
+def find_all_subclasses(full_package: str, class_type,
expected_in_package_name: Optional[str] = None,
+ exclude_class_type=None):
+ import inspect
+ subclasses = set()
+ for global_name, global_object in globals().items():
+ if global_name.startswith(full_package) and
inspect.isclass(global_object):
+ mro = inspect.getmro(global_object)
+ if global_object is not class_type and \
+ class_type in mro and \
+ "example_dags" not in global_name and \
+ (expected_in_package_name is None or expected_in_package_name
in global_name) and \
+ (exclude_class_type is None or exclude_class_type not in mro)
and \
+ global_name not in EXCLUDED_DUPLICATED_OBJECTS and \
+ not is_imported_google_base_hook(global_name) and\
+ global_object.__module__.startswith(full_package):
+ subclasses.add(global_name)
+ return subclasses
+
+
+def get_new_and_moved_objects(objects: Set[str], test_moved_object_dict:
Dict[str, str]):
+ new_objects = []
+ moved_objects = {}
+ for obj in objects:
+ if obj in test_moved_object_dict:
+ moved_objects[obj] = test_moved_object_dict[obj]
+ del test_moved_object_dict[obj]
+ else:
+ new_objects.append(obj)
+ new_objects.sort()
+ return new_objects, moved_objects
+
+
+def strip_package(base_package: str, obj_name: str):
+ if obj_name.startswith(base_package):
+ return obj_name[len(base_package) + 1:]
+ else:
+ return obj_name
+
+
+def convert_obj_name_to_url(prefix: str, obj_name):
+ return prefix + "/".join(obj_name.split(".")[:-1]) + ".py"
+
+
+def get_object_code_link(base_package: str, obj_name: str, tag: str):
+ url_prefix = f'https://github.com/apache/airflow/blob/{tag}/'
+ return f'[{strip_package(base_package,
obj_name)}]({convert_obj_name_to_url(url_prefix, obj_name)})'
+
+
+def convert_new_objects_to_table(obj_list: List[str], full_package_name: str,
object_type: str):
+ from tabulate import tabulate
+ headers = [f"New Airflow 2.0 {object_type}: `{full_package_name}` package"]
+ table = [(get_object_code_link(full_package_name, obj, "master"),) for obj
in obj_list]
+ return tabulate(table, headers=headers, tablefmt="pipe")
+
+
+def convert_moved_objects_to_table(obj_dict: Dict[str, str],
full_package_name: str, object_type: str):
+ from tabulate import tabulate
+ headers = [f"Airflow 2.0 {object_type}: `{full_package_name}` package",
+ f"Airflow 1.10.* previous location (usually `airflow.contrib`)"]
+ table = [
+ (get_object_code_link(full_package_name, obj, "master"),
+ get_object_code_link("airflow.contrib", obj_dict[obj],
"v1-10-stable"))
+ for obj in sorted(obj_dict.keys())
+ ]
+ return tabulate(table, headers=headers, tablefmt="pipe")
+
+
+def get_package_class_summary(full_package_name: str):
+ from airflow.secrets import BaseSecretsBackend
+ from airflow.sensors.base_sensor_operator import BaseSensorOperator
+ from airflow.hooks.base_hook import BaseHook
+ from airflow.models.baseoperator import BaseOperator
+ from typing_extensions import Protocol
+ operators = find_all_subclasses(full_package=full_package_name,
+ class_type=BaseOperator,
+ expected_in_package_name=".operators.",
+ exclude_class_type=BaseSensorOperator)
+ sensors = find_all_subclasses(full_package=full_package_name,
+ class_type=BaseSensorOperator,
+ expected_in_package_name='.sensors.')
+ hooks = find_all_subclasses(full_package=full_package_name,
+ class_type=BaseHook,
+ expected_in_package_name='.hooks.')
+ protocols = find_all_subclasses(full_package=full_package_name,
+ class_type=Protocol)
+ secrets = find_all_subclasses(full_package=full_package_name,
+ class_type=BaseSecretsBackend)
+ new_operators, moved_operators = get_new_and_moved_objects(operators,
MOVED_OPERATORS_DICT)
+ new_sensors, moved_sensors = get_new_and_moved_objects(sensors,
MOVED_SENSORS_DICT)
+ new_hooks, moved_hooks = get_new_and_moved_objects(hooks, MOVED_HOOKS_DICT)
+ new_protocols, moved_protocols = get_new_and_moved_objects(protocols,
MOVED_PROTOCOLS_DICT)
+ new_secrets, moved_secrets = get_new_and_moved_objects(secrets,
MOVED_SECRETS_DICT)
+ class_summary = {
+ "NEW_OPERATORS": new_operators,
+ "MOVED_OPERATORS": moved_operators,
+ "NEW_SENSORS": new_sensors,
+ "MOVED_SENSORS": moved_sensors,
+ "NEW_HOOKS": new_hooks,
+ "MOVED_HOOKS": moved_hooks,
+ "NEW_PROTOCOLS": new_protocols,
+ "MOVED_PROTOCOLS": moved_protocols,
+ "NEW_SECRETS": new_secrets,
+ "MOVED_SECRETS": moved_secrets,
+ }
+ for from_name, to_name, object_type in [
+ ("NEW_OPERATORS", "NEW_OPERATORS_TABLE", "operators"),
+ ("NEW_SENSORS", "NEW_SENSORS_TABLE", "sensors"),
+ ("NEW_HOOKS", "NEW_HOOKS_TABLE", "hooks"),
+ ("NEW_PROTOCOLS", "NEW_PROTOCOLS_TABLE", "protocols"),
+ ("NEW_SECRETS", "NEW_SECRETS_TABLE", "secrets"),
+ ]:
+ class_summary[to_name] =
convert_new_objects_to_table(class_summary[from_name],
+
full_package_name,
+ object_type)
+ for from_name, to_name, object_type in [
+ ("MOVED_OPERATORS", "MOVED_OPERATORS_TABLE", "operators"),
+ ("MOVED_SENSORS", "MOVED_SENSORS_TABLE", "sensors"),
+ ("MOVED_HOOKS", "MOVED_HOOKS_TABLE", "hooks"),
+ ("MOVED_PROTOCOLS", "MOVED_PROTOCOLS_TABLE", "protocols"),
+ ("MOVED_SECRETS", "MOVED_SECRETS_TABLE", "protocols"),
+ ]:
+ class_summary[to_name] =
convert_moved_objects_to_table(class_summary[from_name],
+
full_package_name,
+ object_type)
+ return class_summary
+
+
+def prepare_readme_from_template(template_name: str, context: Dict[str, Any]):
Review comment:
```suggestion
def render_template(template_name: str, context: Dict[str, Any]):
```
There is no logic related with the README.md files in the method code. It
made me wonder when I read the code that used this method
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
