CRZbulabula commented on code in PR #19273:
URL: https://github.com/apache/iotdb/pull/19273#discussion_r4573452608


##########
iotdb-core/ainode/iotdb/ainode/core/model/tirex2/pipeline_tirex2.py:
##########
@@ -0,0 +1,130 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import torch
+
+from iotdb.ainode.core.inference.pipeline.basic_pipeline import 
ForecastPipeline
+from iotdb.ainode.core.model.model_info import ModelInfo
+from iotdb.ainode.core.model.tirex2.types import TimeseriesType
+
+
+class TiRex2Pipeline(ForecastPipeline):
+    """
+    Forecasting pipeline of TiRex-2.
+
+    Each input is one series of one or more targets, forecast jointly, with 
optional past
+    covariates and future-known covariates. The model predicts nine quantiles 
(0.1 to 0.9) of up
+    to 320 future steps in one pass; the pipeline returns the median.
+    """
+
+    def __init__(self, model_info: ModelInfo, **model_kwargs):
+        super().__init__(model_info, **model_kwargs)
+
+    @property
+    def max_output_length(self) -> int:
+        return self.model.future_len
+
+    def _preprocess(self, inputs, **infer_kwargs) -> list[TimeseriesType]:
+        """
+        Convert the validated inputs to TiRex-2 series.
+
+        Parameters
+        ----------
+        inputs : list of dict
+            Each dictionary contains:
+            - 'targets': A tensor of shape (input_length,) or (target_count, 
input_length).
+            - 'past_covariates' (optional): A dictionary of tensors of shape 
(input_length,).
+            - 'future_covariates' (optional): A dictionary of tensors of shape 
(output_length,),
+              whose keys are a subset of 'past_covariates'.
+
+        infer_kwargs:
+            - `output_length` (int): The number of steps to forecast, at most 
320.
+
+        Returns
+        -------
+        list of TimeseriesType
+            One series per input: the targets as a (target_count, 
input_length) tensor, the
+            covariates known only in the past as a (count, input_length) 
tensor, and the
+            covariates also known in the future as a (count, input_length + 
output_length)
+            tensor of their past and future values.
+        """
+        output_length = infer_kwargs.get("output_length", 96)
+        if output_length > self.max_output_length:
+            raise ValueError(

Review Comment:
   Confirmed with a bounded reproduction using the actual pool execution loop, 
batcher, and TiRex-2 pipeline validation (only the model was stubbed with 
`future_len=320`): a 400-step request terminates the execution thread with 
`ValueError`, leaves the finished queue empty, and leaves a subsequent valid 
request queued. Please add coverage through the loaded-model pool path, 
including a valid request after the rejection; the current test only verifies 
that the direct pipeline call raises.



##########
iotdb-core/ainode/iotdb/ainode/core/model/tirex2/component/mlstm_block.py:
##########
@@ -0,0 +1,197 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Copyright (c) NXAI GmbH.
+# SPDX-License-Identifier: Apache-2.0
+
+"""mLSTM layer copied from xLSTM Large, with TiRex-specific tweaks."""
+
+from dataclasses import dataclass
+from typing import Literal
+
+import torch
+import torch.nn as nn
+
+from iotdb.ainode.core.model.tirex2.component import kernel
+from iotdb.ainode.core.model.tirex2.component.norm import MultiHeadLayerNorm
+
+# soft_cap, bias_linspace_init_ and the mLSTMLayerConfig fields are copied 
from xlstm.
+
+
+def soft_cap(
+    values: torch.Tensor, cap_value: float | torch.Tensor | None
+) -> torch.Tensor:
+    """Soft caps a tensor to cap_value with a scaled tanh; no cap when 
cap_value is None."""
+    if cap_value is None:
+        return values
+    return cap_value * torch.tanh(values / cap_value)
+
+
+def bias_linspace_init_(
+    param: torch.Tensor, start: float = 3.4, end: float = 6.0
+) -> torch.Tensor:
+    """Linearly spaced bias init across dimensions."""
+    assert (
+        param.dim() == 1
+    ), f"param must be 1-dimensional (typically a bias), got {param.dim()}"
+    n_dims = param.shape[0]
+    init_vals = torch.linspace(start, end, n_dims)
+    with torch.no_grad():
+        param.copy_(init_vals)
+    return param
+
+
+@dataclass
+class mLSTMLayerConfig:
+    """Configuration of the xLSTM Large mLSTM layer."""
+
+    embedding_dim: int
+    num_heads: int
+    use_bias: bool = False
+    norm_eps: float = 1e-6
+    qk_dim_factor: float = 0.5
+    v_dim_factor: float = 1.0
+    gate_soft_cap: float = 15.0
+    # "auto" runs the Triton kernel for CUDA inputs when Triton is available 
and the native one otherwise.
+    mlstm_kernel: Literal["auto", "native", "triton"] = "auto"
+
+
+class mLSTMLayer(nn.Module):
+    """mLSTM implementation copied from xLSTM 7B."""
+
+    def __init__(self, config: mLSTMLayerConfig):
+        super().__init__()
+        self.config = config
+
+        self.v_dim = int(config.embedding_dim * config.v_dim_factor)
+        self.qk_dim = int(config.embedding_dim * config.qk_dim_factor)
+
+        # Fused input projection of q, k, v and the o, i, f gate 
pre-activations, in that order.
+        # The i and f gates always have a bias; without use_bias the rest of 
the bias starts at zero.
+        self.in_proj_split = (
+            self.qk_dim,
+            self.qk_dim,
+            self.v_dim,
+            self.v_dim,
+            self.config.num_heads,
+            self.config.num_heads,
+        )
+        self.in_proj = nn.Linear(
+            self.config.embedding_dim, sum(self.in_proj_split), bias=True
+        )
+        if not self.config.use_bias:
+            nn.init.zeros_(self.in_proj.bias)
+
+        self.ogate_act_fn = nn.Sigmoid()
+        if config.mlstm_kernel not in ("auto", "native", "triton"):
+            raise ValueError(
+                f"mLSTM kernel must be 'auto', 'native' or 'triton', got 
{config.mlstm_kernel!r}."
+            )
+
+        self.multihead_norm = MultiHeadLayerNorm(
+            num_heads=self.config.num_heads,
+            head_dim=self.v_dim // self.config.num_heads,
+            eps=self.config.norm_eps,
+        )
+        self.out_proj = nn.Linear(
+            in_features=self.v_dim,
+            out_features=self.config.embedding_dim,
+            bias=self.config.use_bias,
+        )
+
+        # Match mLSTMBlock.reset_parameters gate initialisation; i and f are 
the last rows of in_proj.
+        i_rows = slice(-2 * config.num_heads, -config.num_heads)
+        f_rows = slice(-config.num_heads, None)
+        with torch.no_grad():
+            torch.nn.init.zeros_(self.in_proj.weight[f_rows])
+            bias_linspace_init_(self.in_proj.bias[f_rows], start=3.0, end=6.0)
+            torch.nn.init.zeros_(self.in_proj.weight[i_rows])
+            torch.nn.init.normal_(self.in_proj.bias[i_rows], mean=0.0, std=0.1)
+
+    def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs):
+        # Older checkpoints store a separate linear layer per projection
+        legacy = ("q", "k", "v", "ogate_preact", "igate_preact", 
"fgate_preact")
+        if all(f"{prefix}{name}.weight" in state_dict for name in legacy):

Review Comment:
   Confirmed through the actual Hub loader using the tiny published-layout 
checkpoint. Removing either `igate_preact.bias` or `fgate_preact.bias` still 
succeeds with `strict=True`, and the resulting fused `in_proj.bias` differs 
from the original. Please require the learned gate biases before fusion and add 
both cases to the missing-parameter regression test.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to