FrankChen021 commented on code in PR #19687:
URL: https://github.com/apache/druid/pull/19687#discussion_r3695811354
##########
indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/CostBasedAutoScaler.java:
##########
@@ -385,6 +427,16 @@ int computeOptimalTaskCount(CostMetrics metrics)
}
}
+ final double costDropPercent
+ = 100.0 * (currentCost.totalCost() - optimalCost.totalCost()) /
currentCost.totalCost();
+ if (costDropPercent < config.getMinCostDropPercentForScaling()) {
Review Comment:
[P1] Preserve the critical-lag override in the shared cost-drop gate
After the merge from master, critical lag deliberately forces taskCountMax
even when that candidate costs more. This unconditional second gate runs after
the correctly guarded if (!criticalLag) block and returns currentTaskCount
whenever costDropPercent is negative—even with the default minimum of
zero—breaking testCriticalLagJumpsToMaxEvenWhenMaxCostsMore and production
emergency scaling. Guard the shared simulation-compatible gate with
!criticalLag and remove the duplicate earlier check.
##########
web-console/src/dialogs/supervisor-table-action-dialog/auto-scaler-panel/auto-scaler-panel.tsx:
##########
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { FormGroup, NumericInput, Slider } from '@blueprintjs/core';
+import type { ECharts } from 'echarts';
+import * as echarts from 'echarts';
+import React, { useEffect, useMemo, useRef, useState } from 'react';
+
+import { Loader } from '../../../components/loader/loader';
+import { useQueryManager } from '../../../hooks';
+import { Api } from '../../../singletons';
+
+import './auto-scaler-panel.scss';
+
+interface AutoScalerRow {
+ lag: number;
+ taskCount: number;
+}
+
+interface AutoScalerPanelProps {
+ supervisorId: string;
+}
+
+export function getAutoScalerValidationError({
+ taskCountMin,
+ taskCountMax,
+ maxProcessingRatePerTask,
+ optimalTaskIdleRatio,
+ criticalLag,
+ currentTaskCount,
+}: {
+ taskCountMin: number;
+ taskCountMax: number;
+ maxProcessingRatePerTask: number;
+ optimalTaskIdleRatio: number;
+ criticalLag: number;
+ currentTaskCount: number | undefined;
+}): string | undefined {
+ if (taskCountMin > taskCountMax) return 'Minimum task count must not exceed
maximum task count';
+ if (maxProcessingRatePerTask < 100) return 'Max processing rate / task must
be at least 100';
+ if (optimalTaskIdleRatio <= 0 || optimalTaskIdleRatio >= 1) {
+ return 'Optimal task idle ratio must be greater than 0 and less than 1';
+ }
+ if (criticalLag < 1000) return 'Critical lag must be at least 1000';
+ if (
+ currentTaskCount !== undefined &&
+ (currentTaskCount < taskCountMin || currentTaskCount > taskCountMax)
+ ) {
+ return 'Current task count must be within the minimum and maximum task
count';
+ }
+ return undefined;
+}
+
+export const AutoScalerPanel = React.memo(function AutoScalerPanel(props:
AutoScalerPanelProps) {
+ const { supervisorId } = props;
+
+ const [taskCountMin, setTaskCountMin] = useState<number>(1);
+ const [taskCountMax, setTaskCountMax] = useState<number>(10);
+ const [maxProcessingRatePerTask, setMaxProcessingRatePerTask] =
useState<number>(10000);
+ const [optimalTaskIdleRatio, setOptimalTaskIdleRatio] =
useState<number>(0.2);
+ const [lagWeight, setLagWeight] = useState<number>(0.4);
+ // Idle weight is the complement of lag weight; one slider drives both.
+ const idleWeight = Math.round((1 - lagWeight) * 10) / 10;
+ const [criticalLag, setCriticalLag] = useState<number>(100000);
+ // Undefined means "let the server use the supervisor's live task count".
+ const [currentTaskCount, setCurrentTaskCount] = useState<number |
undefined>(undefined);
+
+ const chartContainerRef = useRef<HTMLDivElement | undefined>(undefined);
+ const chartRef = useRef<ECharts | undefined>(undefined);
+ const query = useMemo(
+ () => ({
+ supervisorId,
+ taskCountMin,
+ taskCountMax,
+ maxProcessingRatePerTask,
+ optimalTaskIdleRatio,
+ lagWeight,
+ idleWeight,
+ criticalLag,
+ currentTaskCount,
+ }),
+ [
+ supervisorId,
+ taskCountMin,
+ taskCountMax,
+ maxProcessingRatePerTask,
+ optimalTaskIdleRatio,
+ lagWeight,
+ idleWeight,
+ criticalLag,
+ currentTaskCount,
+ ],
+ );
+ const validationError = getAutoScalerValidationError(query);
+
+ const [dataState] = useQueryManager<
+ {
+ supervisorId: string;
+ taskCountMin: number;
+ taskCountMax: number;
+ maxProcessingRatePerTask: number;
+ optimalTaskIdleRatio: number;
+ lagWeight: number;
+ idleWeight: number;
+ criticalLag: number;
+ currentTaskCount: number | undefined;
+ },
+ AutoScalerRow[]
+ >({
+ query: validationError ? undefined : query,
+ debounceIdle: 300,
+ debounceLoading: 500,
+ processQuery: async (params, signal) => {
+ const resp = await Api.instance.post<{ data: AutoScalerRow[] }>(
+
`/druid/indexer/v1/supervisor/${Api.encodePath(params.supervisorId)}/autoscaler`,
+ {
+ autoScalerStrategy: 'costBased',
+ enableTaskAutoScaler: true,
+ taskCountMin: params.taskCountMin,
+ taskCountMax: params.taskCountMax,
+ optimalTaskIdleRatio: params.optimalTaskIdleRatio,
+ lagWeight: params.lagWeight,
+ idleWeight: params.idleWeight,
+ },
+ {
+ params: {
+ maxProcessingRatePerTask: params.maxProcessingRatePerTask,
+ criticalLag: params.criticalLag,
Review Comment:
[P2] Pass critical lag into the simulated autoscaler config
This value is sent only as a query parameter used to choose the sampled lag
range. Since it is omitted from the CostBasedAutoScalerConfig body,
getCriticalLagThreshold() remains null and the newly merged 75% high-lag
behavior and 100% jump-to-max behavior are disabled. The chart therefore
diverges from the real autoscaler for the critical-lag setting being tuned.
Send criticalLagThreshold: params.criticalLag in the body, or separate and
rename this input if it is intended only as a plot range.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
