bhirsz commented on code in PR #23351:
URL: https://github.com/apache/airflow/pull/23351#discussion_r862319421
##########
tests/always/test_project_structure.py:
##########
@@ -213,76 +299,110 @@ class
TestGoogleProviderProjectStructure(unittest.TestCase):
'airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteDeidentifyTemplateOperator',
'airflow.providers.google.cloud.operators.dlp.CloudDLPListDLPJobsOperator',
'airflow.providers.google.cloud.operators.dlp.CloudDLPRedactImageOperator',
+
'airflow.providers.google.cloud.transfers.cassandra_to_gcs.CassandraToGCSOperator',
+
'airflow.providers.google.cloud.transfers.adls_to_gcs.ADLSToGCSOperator',
+
'airflow.providers.google.cloud.transfers.bigquery_to_mysql.BigQueryToMySqlOperator',
+
'airflow.providers.google.cloud.transfers.sql_to_gcs.BaseSQLToGCSOperator',
+
'airflow.providers.google.cloud.operators.vertex_ai.endpoint_service.GetEndpointOperator',
+
'airflow.providers.google.cloud.operators.vertex_ai.auto_ml.AutoMLTrainingJobBaseOperator',
+
'airflow.providers.google.cloud.operators.vertex_ai.endpoint_service.UpdateEndpointOperator',
+
'airflow.providers.google.cloud.operators.vertex_ai.batch_prediction_job.'
+ 'GetBatchPredictionJobOperator',
}
- def test_missing_example_for_operator(self):
- """
- Assert that all operators defined under operators, sensors and
transfers directories
- are used in any of the example dags
- """
- all_operators = set()
- services = set()
- for resource_type in ["operators", "sensors", "transfers"]:
- operator_files = set(
- self.find_resource_files(top_level_directory="airflow",
resource_type=resource_type)
- )
- for filepath in operator_files:
- service_name = os.path.basename(filepath)[: -(len(".py"))]
- if service_name in self.MISSING_EXAMPLE_DAGS:
- continue
- services.add(service_name)
- operators_paths =
set(get_classes_from_file(f"{ROOT_FOLDER}/{filepath}"))
- all_operators.update(operators_paths)
-
- for service in services:
- example_dags = self.examples_for_service(service)
- example_paths = {
- path for example_dag in example_dags for path in
get_imports_from_file(example_dag)
- }
- all_operators -= example_paths
+ # These operators should not have assets
+ MISSING_ASSETS_FOR_OPERATORS = {
+
'airflow.providers.google.cloud.operators.automl.AutoMLDeleteDatasetOperator',
+
'airflow.providers.google.cloud.operators.automl.AutoMLDeleteModelOperator',
+
'airflow.providers.google.cloud.operators.bigquery.BigQueryCheckOperator',
+
'airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteDatasetOperator',
+
'airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteTableOperator',
+
'airflow.providers.google.cloud.operators.bigquery.BigQueryIntervalCheckOperator',
+
'airflow.providers.google.cloud.operators.bigquery.BigQueryValueCheckOperator',
+
'airflow.providers.google.cloud.operators.bigquery_dts.BigQueryDeleteDataTransferConfigOperator',
+
'airflow.providers.google.cloud.operators.bigtable.BigtableDeleteInstanceOperator',
+
'airflow.providers.google.cloud.operators.bigtable.BigtableDeleteTableOperator',
+
'airflow.providers.google.cloud.operators.cloud_build.CloudBuildDeleteBuildTriggerOperator',
+
'airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreDeleteInstanceOperator',
+ 'airflow.providers.google.cloud.operators.cloud_memorystore.'
+ 'CloudMemorystoreMemcachedDeleteInstanceOperator',
+
'airflow.providers.google.cloud.operators.cloud_sql.CloudSQLBaseOperator',
+
'airflow.providers.google.cloud.operators.cloud_sql.CloudSQLDeleteInstanceDatabaseOperator',
+
'airflow.providers.google.cloud.operators.cloud_sql.CloudSQLDeleteInstanceOperator',
+
'airflow.providers.google.cloud.operators.cloud_storage_transfer_service.'
+ 'CloudDataTransferServiceDeleteJobOperator',
+
'airflow.providers.google.cloud.operators.cloud_storage_transfer_service.'
+ 'CloudDataTransferServiceGetOperationOperator',
+
'airflow.providers.google.cloud.operators.cloud_storage_transfer_service.'
+ 'CloudDataTransferServiceListOperationsOperator',
+
'airflow.providers.google.cloud.operators.cloud_storage_transfer_service.'
+ 'CloudDataTransferServicePauseOperationOperator',
+
'airflow.providers.google.cloud.operators.cloud_storage_transfer_service.'
+ 'CloudDataTransferServiceResumeOperationOperator',
+
'airflow.providers.google.cloud.operators.compute.ComputeEngineBaseOperator',
+
'airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogDeleteEntryGroupOperator',
+
'airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogDeleteEntryOperator',
+
'airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogDeleteTagOperator',
+
'airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogDeleteTagTemplateFieldOperator',
+
'airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogDeleteTagTemplateOperator',
+
'airflow.providers.google.cloud.operators.datafusion.CloudDataFusionDeleteInstanceOperator',
+
'airflow.providers.google.cloud.operators.datafusion.CloudDataFusionDeletePipelineOperator',
+
'airflow.providers.google.cloud.operators.dataproc.DataprocDeleteBatchOperator',
+
'airflow.providers.google.cloud.operators.dataproc.DataprocDeleteClusterOperator',
+
'airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreDeleteBackupOperator',
+
'airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreDeleteServiceOperator',
+
'airflow.providers.google.cloud.operators.datastore.CloudDatastoreBeginTransactionOperator',
+
'airflow.providers.google.cloud.operators.datastore.CloudDatastoreDeleteOperationOperator',
+
'airflow.providers.google.cloud.operators.datastore.CloudDatastoreGetOperationOperator',
+
'airflow.providers.google.cloud.operators.datastore.CloudDatastoreRollbackOperator',
+
'airflow.providers.google.cloud.operators.datastore.CloudDatastoreRunQueryOperator',
+
'airflow.providers.google.cloud.operators.dlp.CloudDLPDeidentifyContentOperator',
+
'airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteDLPJobOperator',
+
'airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteDeidentifyTemplateOperator',
+
'airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteInspectTemplateOperator',
+
'airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteJobTriggerOperator',
+
'airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteStoredInfoTypeOperator',
+
'airflow.providers.google.cloud.operators.dlp.CloudDLPInspectContentOperator',
+
'airflow.providers.google.cloud.operators.dlp.CloudDLPRedactImageOperator',
+
'airflow.providers.google.cloud.operators.dlp.CloudDLPReidentifyContentOperator',
+
'airflow.providers.google.cloud.operators.functions.CloudFunctionDeleteFunctionOperator',
+ 'airflow.providers.google.cloud.operators.gcs.GCSDeleteBucketOperator',
+
'airflow.providers.google.cloud.operators.gcs.GCSDeleteObjectsOperator',
+
'airflow.providers.google.cloud.operators.kubernetes_engine.GKEDeleteClusterOperator',
+
'airflow.providers.google.cloud.operators.mlengine.MLEngineDeleteModelOperator',
+
'airflow.providers.google.cloud.operators.mlengine.MLEngineDeleteVersionOperator',
+
'airflow.providers.google.cloud.operators.pubsub.PubSubDeleteSubscriptionOperator',
+
'airflow.providers.google.cloud.operators.pubsub.PubSubDeleteTopicOperator',
+
'airflow.providers.google.cloud.operators.spanner.SpannerDeleteDatabaseInstanceOperator',
+
'airflow.providers.google.cloud.operators.spanner.SpannerDeleteInstanceOperator',
+
'airflow.providers.google.cloud.operators.stackdriver.StackdriverDeleteAlertOperator',
+
'airflow.providers.google.cloud.operators.stackdriver.StackdriverDeleteNotificationChannelOperator',
+
'airflow.providers.google.cloud.operators.tasks.CloudTasksQueueDeleteOperator',
+
'airflow.providers.google.cloud.operators.tasks.CloudTasksTaskDeleteOperator',
+
'airflow.providers.google.cloud.operators.translate.CloudTranslateTextOperator',
+
'airflow.providers.google.cloud.operators.translate_speech.CloudTranslateSpeechOperator',
+
'airflow.providers.google.cloud.operators.vision.CloudVisionDeleteProductOperator',
+
'airflow.providers.google.cloud.operators.vision.CloudVisionDeleteProductSetOperator',
+
'airflow.providers.google.cloud.operators.vision.CloudVisionDeleteReferenceImageOperator',
+
'airflow.providers.google.cloud.operators.workflows.WorkflowsDeleteWorkflowOperator',
+ 'airflow.providers.google.marketing_platform.sensors.campaign_manager.'
+ 'GoogleCampaignManagerReportSensor',
+ 'airflow.providers.google.marketing_platform.sensors.display_video.'
+ 'GoogleDisplayVideo360GetSDFDownloadOperationSensor',
+ 'airflow.providers.google.marketing_platform.sensors.display_video.'
+ 'GoogleDisplayVideo360ReportSensor',
+
'airflow.providers.google.marketing_platform.sensors.search_ads.GoogleSearchAdsReportSensor',
+ }
- all_operators -= self.MISSING_EXAMPLES_FOR_OPERATORS
- all_operators -= self.DEPRECATED_OPERATORS
- all_operators -= self.BASE_OPERATORS
- assert set() == all_operators
-
- @parameterized.expand(
- itertools.product(["_system.py", "_system_helper.py"], ["operators",
"sensors", "transfers"])
- )
- def test_detect_invalid_system_tests(self, resource_type, filename_suffix):
- operators_tests =
self.find_resource_files(top_level_directory="tests",
resource_type=resource_type)
- operators_files =
self.find_resource_files(top_level_directory="airflow",
resource_type=resource_type)
-
- files = {f for f in operators_tests if f.endswith(filename_suffix)}
-
- expected_files = (f"tests/{f[8:]}" for f in operators_files)
- expected_files = (f.replace(".py", filename_suffix).replace("/test_",
"/") for f in expected_files)
- expected_files =
{f'{f.rpartition("/")[0]}/test_{f.rpartition("/")[2]}' for f in expected_files}
-
- assert set() == files - expected_files
-
- @staticmethod
- def find_resource_files(
- top_level_directory: str = "airflow",
- department: str = "*",
- resource_type: str = "*",
- service: str = "*",
- ):
- python_files = glob.glob(
-
f"{ROOT_FOLDER}/{top_level_directory}/providers/google/{department}/{resource_type}/{service}.py"
- )
- # Make path relative
- resource_files = (os.path.relpath(f, ROOT_FOLDER) for f in
python_files)
- # Exclude __init__.py and pycache
- resource_files = (f for f in resource_files if not
f.endswith("__init__.py"))
- return resource_files
- @staticmethod
- def examples_for_service(service_name):
- yield from glob.glob(
-
f"{ROOT_FOLDER}/airflow/providers/google/*/example_dags/example_{service_name}*.py"
- )
- yield from
glob.glob(f"{ROOT_FOLDER}/tests/system/providers/google/{service_name}/example_*.py")
+class TestElasticsearchProviderProjectStructure(ExampleCoverageTest):
+ PROVIDER = "elasticsearch"
+ OPERATOR_DIRS = {"hooks"}
Review Comment:
I agree, I'm not satisfied with current naming (just took over the old
naming). Same goes for method names such as list_of_operators etc. I will think
about it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
