potiuk commented on a change in pull request #6692: [AIRFLOW-6130] Make
Cassandra to GCS operator pylint compatible
URL: https://github.com/apache/airflow/pull/6692#discussion_r352285880
##########
File path: airflow/operators/cassandra_to_gcs.py
##########
@@ -209,29 +201,38 @@ def _write_local_schema_file(self, cursor):
schema = []
tmp_schema_file_handle = NamedTemporaryFile(delete=True)
- for name, type in zip(cursor.column_names, cursor.column_types):
- schema.append(self.generate_schema_dict(name, type))
+ for name, type_ in zip(cursor.column_names, cursor.column_types):
+ schema.append(self.generate_schema_dict(name, type_))
json_serialized_schema = json.dumps(schema).encode('utf-8')
tmp_schema_file_handle.write(json_serialized_schema)
return {self.schema_filename: tmp_schema_file_handle}
- def _upload_to_gcs(self, files_to_upload):
+ def _upload_to_gcs(self, files_to_upload: Dict):
hook = GoogleCloudStorageHook(
google_cloud_storage_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to)
- for object, tmp_file_handle in files_to_upload.items():
- hook.upload(self.bucket, object, tmp_file_handle.name,
'application/json', self.gzip)
+ for obj, tmp_file_handle in files_to_upload.items():
+ hook.upload(
+ bucket_name=self.bucket,
+ object_name=obj,
+ filename=tmp_file_handle.name,
+ mime_type='application/json',
+ gzip=self.gzip
+ )
@classmethod
- def generate_data_dict(cls, names, values):
- row_dict = {}
- for name, value in zip(names, values):
- row_dict.update({name: cls.convert_value(name, value)})
- return row_dict
+ def generate_data_dict(cls, names: Iterable[str], values: Any) -> Dict:
+ """
+ Generates data structure that will be stored as file in GCS.
+ """
+ return {n: cls.convert_value(v) for n, v in zip(names, values)}
@classmethod
- def convert_value(cls, name, value):
+ def convert_value(cls, value): # pylint:disable=too-many-return-statements
Review comment:
```suggestion
def convert_value(cls, value: Optional[Any]) -> Optional[Any]: #
pylint:disable=too-many-return-statements
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
