CRZbulabula commented on code in PR #19272:
URL: https://github.com/apache/iotdb/pull/19272#discussion_r4573452780
##########
iotdb-core/ainode/iotdb/ainode/core/inference/pool_controller.py:
##########
@@ -201,6 +201,15 @@ def _worker_loop(self):
self._task_queue.task_done()
def _load_one_model_task(self, model_id: str, device_id_list:
list[torch.device]):
+ model_info = self._model_manager.get_model_info(model_id)
+ if not model_info.download_on_start:
+ # Built-in models that are not downloaded at start are downloaded
when loaded
+ if not self._model_manager.download_builtin_model(model_id):
Review Comment:
Confirmed with an event-controlled download: a task queued behind `LOAD
MODEL timestral` does not execute until the download completes. This is the
controller's only management worker, and `stop()` joins it before closing
existing pools, so a slow download also delays unrelated unloads and shutdown.
Please move the transfer off this queue and make shutdown cancellation bounded,
while preserving ordering for the affected model.
##########
iotdb-core/ainode/iotdb/ainode/core/model/timestral/pipeline_timestral.py:
##########
@@ -0,0 +1,140 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import torch
+
+from iotdb.ainode.core.inference.pipeline.basic_pipeline import
ForecastPipeline
+from iotdb.ainode.core.model.model_info import ModelInfo
+from iotdb.ainode.core.model.timestral.forecaster import TimestralForecaster
+
+
+class TimestralPipeline(ForecastPipeline):
+ """
+ Forecasting pipeline of Timestral.
+
+ Each input is one series of one or more targets, forecast jointly, with
optional past
+ covariates and future-known covariates. The model predicts 21 quantiles
(0.01 to 0.99);
+ horizons longer than one decode block (1024 steps) are forecast block by
block, feeding
+ each block's median back as context. The pipeline returns the median.
+ """
+
+ def __init__(self, model_info: ModelInfo, **model_kwargs):
+ super().__init__(model_info, **model_kwargs)
+ self.forecaster = TimestralForecaster(self.model)
+
+ def _preprocess(self, inputs, **infer_kwargs) -> list[dict]:
+ """
+ Convert the validated inputs to the arguments of the forecaster.
+
+ Parameters
+ ----------
+ inputs : list of dict
+ Each dictionary contains:
+ - 'targets': A tensor of shape (input_length,) or (target_count,
input_length).
+ - 'past_covariates' (optional): A dictionary of tensors of shape
(input_length,).
+ - 'future_covariates' (optional): A dictionary of tensors of shape
(output_length,),
+ whose keys are a subset of 'past_covariates'.
+
+ Returns
+ -------
+ list of dict
+ One dictionary per input, with:
+ - 'context': The targets, a (1, target_count, input_length) tensor.
+ - 'past_covariates': The covariates known only in the past, a
+ (1, count, input_length) tensor, or None.
+ - 'known_covariates': The covariates also known in the future, a
+ (1, count, input_length + output_length) tensor of their past
and future
+ values, or None.
+ """
+ series = []
+ for item in inputs:
+ targets = item["targets"]
+ if targets.ndim == 1:
+ targets = targets.unsqueeze(0)
+ past_covariates = item.get("past_covariates") or {}
+ future_covariates = item.get("future_covariates") or {}
+ past_only = [k for k in past_covariates if k not in
future_covariates]
+ known = [k for k in past_covariates if k in future_covariates]
+ series.append(
+ {
+ "context": targets.float().unsqueeze(0),
+ "past_covariates": (
+ torch.stack([past_covariates[k] for k in past_only])
+ .float()
+ .unsqueeze(0)
+ if past_only
+ else None
+ ),
+ "known_covariates": (
+ torch.stack(
+ [
+ torch.cat([past_covariates[k],
future_covariates[k]])
+ for k in known
+ ]
+ )
+ .float()
+ .unsqueeze(0)
+ if known
+ else None
+ ),
+ }
+ )
+ return series
+
+ def forecast(self, inputs: list[dict], **infer_kwargs) ->
list[torch.Tensor]:
+ """
+ Forecast the series, batching those of the same shape without
covariates.
+
+ Returns
+ -------
+ list of torch.Tensor
+ One tensor per series, of shape (target_count, output_length,
quantile_count).
+ """
+ output_length = infer_kwargs.get("output_length", 96)
+ outputs: list[torch.Tensor | None] = [None] * len(inputs)
+ batches: dict[tuple, list[int]] = {}
+ for i, item in enumerate(inputs):
+ if item["past_covariates"] is None and item["known_covariates"] is
None:
+ batches.setdefault(tuple(item["context"].shape), []).append(i)
+ else:
+ batches[("series", i)] = [i]
Review Comment:
I checked the actual mask builder with meta tensors: one 100-target series
with 1,056 patch slots produces a dense boolean mask of shape `[1056, 1, 100,
100]` (10,560,000 bytes), while ten such series produce `[1056, 1, 1000, 1000]`
(1,056,000,000 bytes). That is 100 times the mask memory for ten times the
input; the normal request scheduler can select ten requests at once. Please
preserve a separate batch dimension for independent series or process them
separately/in bounded batches, and add a regression covering multiple
concurrent requests.
##########
iotdb-core/ainode/iotdb/ainode/core/model/model_storage.py:
##########
@@ -132,55 +135,112 @@ def _process_builtin_model_directory(self, model_dir:
str, model_id: str):
if existing_model.state in (ModelStates.ACTIVATING,
ModelStates.ACTIVE):
return
+ model_info = BUILTIN_HF_TRANSFORMERS_MODEL_MAP[model_id]
+ self._models[ModelCategory.BUILTIN.value][model_id] = model_info
+ # A model whose weights are not downloaded at start stays INACTIVE
until it is
+ # loaded, which downloads them (see download_builtin_model)
+ if not model_info.download_on_start and not
self._builtin_files_exist(
+ model_dir
+ ):
+ model_info.state = ModelStates.INACTIVE
+ logger.info(
+ f"Model {model_id} is not downloaded at start; "
+ f"LOAD MODEL {model_id} downloads it from
{model_info.repo_id}."
+ )
+ return
+
# If model not exists or is INACTIVE, we'll try to update its info
and download its weights
- self._models[ModelCategory.BUILTIN.value][model_id] = (
- BUILTIN_HF_TRANSFORMERS_MODEL_MAP[model_id]
- )
- self._models[ModelCategory.BUILTIN.value][
- model_id
- ].state = ModelStates.ACTIVATING
-
- def _download_model_if_necessary() -> bool:
- """Returns: True if the model is existed or downloaded
successfully, False otherwise."""
- repo_id = BUILTIN_HF_TRANSFORMERS_MODEL_MAP[model_id].repo_id
- weights_path = os.path.join(model_dir, MODEL_SAFETENSORS)
- config_path = os.path.join(model_dir, CONFIG_JSON)
- if not os.path.exists(weights_path):
- try:
- hf_hub_download(
- repo_id=repo_id,
- filename=MODEL_SAFETENSORS,
- local_dir=model_dir,
- )
- except Exception as e:
- logger.error(
- f"Failed to download model weights from HuggingFace:
{e}"
- )
- return False
- if not os.path.exists(config_path):
- try:
- hf_hub_download(
- repo_id=repo_id,
- filename=CONFIG_JSON,
- local_dir=model_dir,
- )
- except Exception as e:
- logger.error(
- f"Failed to download model config from HuggingFace:
{e}"
- )
- return False
- return True
+ model_info.state = ModelStates.ACTIVATING
- future = self._executor.submit(_download_model_if_necessary)
+ future = self._executor.submit(
+ self._download_builtin_model_files, model_id, model_dir
+ )
future.add_done_callback(
lambda f, mid=model_id: self._callback_model_download_result(f,
mid)
)
+ @staticmethod
+ def _builtin_files_exist(model_dir: str) -> bool:
+ return os.path.exists(
+ os.path.join(model_dir, MODEL_SAFETENSORS)
+ ) and os.path.exists(os.path.join(model_dir, CONFIG_JSON))
+
+ def _download_builtin_model_files(self, model_id: str, model_dir: str) ->
bool:
+ """Returns: True if the model is existed or downloaded successfully,
False otherwise."""
+ repo_id = BUILTIN_HF_TRANSFORMERS_MODEL_MAP[model_id].repo_id
+ weights_path = os.path.join(model_dir, MODEL_SAFETENSORS)
+ config_path = os.path.join(model_dir, CONFIG_JSON)
+ if not os.path.exists(weights_path):
+ try:
+ hf_hub_download(
+ repo_id=repo_id,
+ filename=MODEL_SAFETENSORS,
+ local_dir=model_dir,
+ )
+ except Exception as e:
+ logger.error(f"Failed to download model weights from
HuggingFace: {e}")
+ return False
+ if not os.path.exists(config_path):
+ try:
+ hf_hub_download(
+ repo_id=repo_id,
+ filename=CONFIG_JSON,
+ local_dir=model_dir,
+ )
+ except Exception as e:
+ logger.error(f"Failed to download model config from
HuggingFace: {e}")
+ return False
+ return True
+
+ def download_builtin_model(self, model_id: str) -> bool:
+ """
+ Download the weights of a built-in model that is not downloaded at
start, and wait
+ until the download ends.
+ Args:
+ model_id (str): Unique identifier for the model.
+ Returns:
+ bool: True if the model is ready to use, False if the download
failed.
+ Raises:
+ ModelNotExistException: If the model_id is not a built-in model.
+ """
+ with self._download_lock:
+ with self._lock_pool.get_lock(model_id).write_lock():
+ model_info =
self._models[ModelCategory.BUILTIN.value].get(model_id)
+ if (
+ model_info is None
+ or model_id not in BUILTIN_HF_TRANSFORMERS_MODEL_MAP
+ ):
+ raise ModelNotExistException(model_id)
+ if model_info.state == ModelStates.ACTIVE:
+ return True
+ model_info.state = ModelStates.ACTIVATING
+ logger.info(
+ f"Downloading model {model_id} from HuggingFace:
{model_info.repo_id}"
+ )
+ model_dir = os.path.join(
+ self._models_dir, ModelCategory.BUILTIN.value, model_id
+ )
+ succeeded = False
+ try:
+ succeeded = self._download_builtin_model_files(model_id,
model_dir)
+ finally:
+ self._set_model_download_result(model_id, succeeded)
+ return succeeded
Review Comment:
Reproduced with a malformed local `config.json`:
`download_builtin_model("timestral")` returns `True` even though
`_set_model_download_result()` leaves the model `INACTIVE`. The controller
consequently proceeds to create a pool whose checkpoint cannot load. Please
propagate configuration-processing failure into the returned result and add a
regression test for malformed JSON.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]