turbaszek commented on a change in pull request #8631:
URL: https://github.com/apache/airflow/pull/8631#discussion_r422863535
##########
File path: airflow/providers/google/cloud/hooks/bigquery.py
##########
@@ -598,33 +597,86 @@ def create_external_table(self, # pylint:
disable=too-many-locals,too-many-argu
],
'googleSheetsOptions': ['skipLeadingRows']
}
-
if source_format in src_fmt_to_param_mapping.keys():
- valid_configs = src_fmt_to_configs_mapping[
- src_fmt_to_param_mapping[source_format]
- ]
-
+ valid_configs =
src_fmt_to_configs_mapping[src_fmt_to_param_mapping[source_format]]
src_fmt_configs = _validate_src_fmt_configs(source_format,
src_fmt_configs, valid_configs,
backward_compatibility_configs)
+ external_config_api_repr[src_fmt_to_param_mapping[source_format]]
= src_fmt_configs
-
table_resource['externalDataConfiguration'][src_fmt_to_param_mapping[
- source_format]] = src_fmt_configs
+ # build external config
+ external_config =
ExternalConfig.from_api_repr(external_config_api_repr)
+ if schema_fields:
+ external_config.schema = [SchemaField.from_api_repr(f) for f in
schema_fields]
+ if max_bad_records:
+ external_config.max_bad_records = max_bad_records
+ # build table definition
+ table = Table(
+
table_ref=TableReference.from_string(external_project_dataset_table, project_id)
+ )
+ table.external_data_configuration = external_config
if labels:
- table_resource['labels'] = labels
+ table.labels = labels
if encryption_configuration:
- table_resource["encryptionConfiguration"] =
encryption_configuration
+ table.encryption_configuration =
EncryptionConfiguration.from_api_repr(encryption_configuration)
+
+ self.log.info('Creating external table: %s',
external_project_dataset_table)
+ self.create_empty_table(table_resource=table.to_api_repr(),
project_id=project_id, location=location)
+ self.log.info('External table created successfully: %s',
external_project_dataset_table)
+
+ @GoogleBaseHook.fallback_to_default_project_id
+ def update_table(
+ self,
+ table_resource: Dict[str, Any],
+ fields: Optional[List[str]] = None,
+ dataset_id: Optional[str] = None,
+ table_id: Optional[str] = None,
+ project_id: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ """
+ Change some fields of a table.
+
+ Use ``fields`` to specify which fields to update. At least one field
+ must be provided. If a field is listed in ``fields`` and is ``None``
+ in ``table``, the field value will be deleted.
+
+ If ``table.etag`` is not ``None``, the update will only succeed if
+ the table on the server has the same ETag. Thus reading a table with
+ ``get_table``, changing its fields, and then passing it to
+ ``update_table`` will ensure that the changes will only be saved if
+ no modifications to the table occurred since the read.
- service.tables().insert( # pylint: disable=no-member
- projectId=project_id,
- datasetId=dataset_id,
- body=table_resource
- ).execute(num_retries=self.num_retries)
+ :param project_id: The project to create the table into.
+ :type project_id: str
+ :param dataset_id: The dataset to create the table into.
+ :type dataset_id: str
+ :param table_id: The Name of the table to be created.
+ :type table_id: str
+ :param table_resource: Table resource as described in documentation:
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#Table
+ The table has to contain ``tableReference`` or ``project_id``,
``datset_id`` and ``table_id``
+ have to be provided.
+ :type table_resource: Dict[str, Any]
+ :param fields: The fields of ``table`` to change, spelled as the Table
+ properties (e.g. "friendly_name").
+ :type fields: List[str]
+ """
+ fields = fields or list(table_resource.keys())
+ table_resource = self._resolve_table_reference(
+ table_resource=table_resource,
+ project_id=project_id,
+ dataset_id=dataset_id,
+ table_id=table_id
+ )
- self.log.info('External table created successfully: %s',
- external_project_dataset_table)
+ table = Table.from_api_repr(table_resource)
+ self.log.info('Updating table %s.%s.%s', project_id, dataset_id,
table_id)
Review comment:
Good catch, user can provide table resource
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
