utkarsharma2 commented on code in PR #36085:
URL: https://github.com/apache/airflow/pull/36085#discussion_r1417256443
##########
airflow/providers/weaviate/operators/weaviate.py:
##########
@@ -51,21 +57,38 @@ def __init__(
self,
conn_id: str,
class_name: str,
- input_json: list[dict[str, Any]],
+ input_json: list[dict[str, Any]] | pd.DataFrame | None = None,
+ input_data: list[dict[str, Any]] | pd.DataFrame | None = None,
+ vector_col: str = "Vector",
**kwargs: Any,
) -> None:
self.batch_params = kwargs.pop("batch_params", {})
self.hook_params = kwargs.pop("hook_params", {})
super().__init__(**kwargs)
self.class_name = class_name
self.conn_id = conn_id
- self.input_json = input_json
+ self.vector_col = vector_col
+ self.input_data = input_data
+ if input_json:
+ warnings.warn(
+ "Passing 'input_json' to WeaviateIngestOperator is deprecated
and"
+ " you should use 'input_data' instead",
+ AirflowProviderDeprecationWarning,
+ )
+ self.input_data = input_json
+ if self.input_data is None:
+ raise ValueError("Either input_json or input_data is required")
@cached_property
def hook(self) -> WeaviateHook:
"""Return an instance of the WeaviateHook."""
return WeaviateHook(conn_id=self.conn_id, **self.hook_params)
def execute(self, context: Context) -> None:
- self.log.debug("Input json: %s", self.input_json)
- self.hook.batch_data(self.class_name, self.input_json,
**self.batch_params)
+ self.log.debug("Input data: %s", self.input_data)
+ self.hook.batch_data(
+ self.class_name,
+ self.input_data, # type: ignore
Review Comment:
@uranusjr I added `# type: ignore` because mypy was assuming the type of
`self.input_data` to be `list[dict[str, Any]] | pd.DataFrame | None` but we
have ensured in the `__init__()` method that it's not None. Is there a better
way of dealing with it?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
