This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new cf84e5f1b09 fix: Web console: compaction of PB scale datasource
returning a bigint that crashes the Datasources view and get Claude to fix all
other issues like it. (#20384)
cf84e5f1b09 is described below
commit cf84e5f1b09ce3b05c2f5c015e5259bf66b7a7e6
Author: Vadim Ogievetsky <[email protected]>
AuthorDate: Mon Sep 21 23:31:58 2026 -0700
fix: Web console: compaction of PB scale datasource returning a bigint that
crashes the Datasources view and get Claude to fix all other issues like it.
(#20384)
* fix compaction status
* fix the rest of the issues
* address review feedback
Only coerce measure columns in bigIntsToNumbers so BIGINT dimension
values above Number.MAX_SAFE_INTEGER keep their exact value, instead of
rounding and merging distinct chart categories / breaking click-to-filter.
Render the Availability cell for BigInt segment counts rather than
returning '-'. Widen the count fields on DatasourceQueryResultRow to
NumberLike and coerce once, so a BigInt count keeps its percentage,
segment link, and zero-replica status.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
---------
Co-authored-by: Claude Opus 5 (1M context) <[email protected]>
---
.../src/components/segment-timeline/interval.ts | 17 ++-
.../segment-timeline/segment-bar-chart-render.tsx | 11 +-
.../compaction-status/compaction-status.ts | 19 ++--
web-console/src/druid-models/stages/stages.ts | 126 ++++++++++++---------
.../supervisor-status/supervisor-status.ts | 59 +++++-----
web-console/src/utils/general.tsx | 25 ++++
.../__snapshots__/datasources-view.spec.tsx.snap | 4 +-
.../views/datasources-view/datasources-view.tsx | 79 +++++++------
.../modules/bar-chart-module/bar-chart-module.tsx | 5 +-
.../multi-axis-chart-module.tsx | 6 +-
.../modules/pie-chart-module/pie-chart-module.tsx | 10 +-
.../time-chart-module/time-chart-module.tsx | 2 +-
.../src/views/segments-view/segments-view.tsx | 2 +-
.../src/views/services-view/services-view.tsx | 14 +--
.../execution-stages-pane.tsx | 14 ++-
15 files changed, 242 insertions(+), 151 deletions(-)
diff --git a/web-console/src/components/segment-timeline/interval.ts
b/web-console/src/components/segment-timeline/interval.ts
index 833f01251cf..67f4684383c 100644
--- a/web-console/src/components/segment-timeline/interval.ts
+++ b/web-console/src/components/segment-timeline/interval.ts
@@ -19,6 +19,7 @@
import type { Duration } from 'chronoshift';
import { sum } from 'd3-array';
+import type { NumberLike } from '../../utils';
import { formatBytes, formatInteger } from '../../utils';
export type IntervalStat = 'segments' | 'size' | 'rows';
@@ -42,16 +43,16 @@ export function getIntervalStatTitle(intervalStat:
IntervalStat): string {
}
export function aggregateSegmentStats(
- xs: readonly Record<IntervalStat, number>[],
+ xs: readonly Record<IntervalStat, NumberLike>[],
): Record<IntervalStat, number> {
return {
- segments: sum(xs, s => s.segments),
- size: sum(xs, s => s.size),
- rows: sum(xs, s => s.rows),
+ segments: sum(xs, s => Number(s.segments)),
+ size: sum(xs, s => Number(s.size)),
+ rows: sum(xs, s => Number(s.rows)),
};
}
-export function formatIntervalStat(stat: IntervalStat, n: number) {
+export function formatIntervalStat(stat: IntervalStat, n: NumberLike) {
switch (stat) {
case 'segments':
case 'rows':
@@ -65,7 +66,7 @@ export function formatIntervalStat(stat: IntervalStat, n:
number) {
}
}
-export interface IntervalRow extends Record<IntervalStat, number> {
+export interface IntervalRow extends Record<IntervalStat, NumberLike> {
start: Date;
end: Date;
datasource: string;
@@ -73,7 +74,11 @@ export interface IntervalRow extends Record<IntervalStat,
number> {
originalTimeSpan: Duration;
}
+// Unlike IntervalRow, the stats here have been through aggregateSegmentStats
so they are plain numbers
export interface TrimmedIntervalRow extends IntervalRow {
+ segments: number;
+ size: number;
+ rows: number;
shownDays: number;
normalized: Record<IntervalStat, number>;
}
diff --git
a/web-console/src/components/segment-timeline/segment-bar-chart-render.tsx
b/web-console/src/components/segment-timeline/segment-bar-chart-render.tsx
index 945b8703fc3..0cd9cdfa820 100644
--- a/web-console/src/components/segment-timeline/segment-bar-chart-render.tsx
+++ b/web-console/src/components/segment-timeline/segment-bar-chart-render.tsx
@@ -271,12 +271,17 @@ export const SegmentBarChartRender = function
SegmentBarChartRender(
const averageRowSizeByDatasource = groupByAsMap(
shownIntervalRows.filter(intervalRow => intervalRow.size > 0 &&
intervalRow.rows > 0),
intervalRow => intervalRow.datasource,
- intervalRows => sum(intervalRows, d => d.size) / sum(intervalRows, d =>
d.rows),
+ intervalRows =>
+ sum(intervalRows, d => Number(d.size)) / sum(intervalRows, d =>
Number(d.rows)),
);
const trimDuration = new Duration(trimGranularity);
const trimmedIntervalRows = shownIntervalRows.map(intervalRow => {
- const { start, end, segments, size, rows } = intervalRow;
+ const { start, end } = intervalRow;
+ // These come off the JSON parser, so they can be BigInt on large
clusters
+ const segments = Number(intervalRow.segments);
+ const size = Number(intervalRow.size);
+ const rows = Number(intervalRow.rows);
const startTrimmed = trimDuration.floor(start, Timezone.UTC);
let endTrimmed = trimDuration.ceil(end, Timezone.UTC);
@@ -293,6 +298,8 @@ export const SegmentBarChartRender = function
SegmentBarChartRender(
start: startTrimmed,
end: endTrimmed,
shownDays,
+ segments,
+ rows,
size: shownSize,
normalized: {
size: shownSize / shownDays,
diff --git
a/web-console/src/druid-models/compaction-status/compaction-status.ts
b/web-console/src/druid-models/compaction-status/compaction-status.ts
index 5712eb958b0..785ff753c50 100644
--- a/web-console/src/druid-models/compaction-status/compaction-status.ts
+++ b/web-console/src/druid-models/compaction-status/compaction-status.ts
@@ -16,6 +16,7 @@
* limitations under the License.
*/
+import type { NumberLike } from '../../utils';
import { formatBytesCompact, pluralIfNeeded } from '../../utils';
import type { CompactionConfig } from '../compaction-config/compaction-config';
import { compactionConfigHasLegacyInputSegmentSizeBytesSet } from
'../compaction-config/compaction-config';
@@ -27,15 +28,15 @@ function capitalizeFirst(str: string): string {
export interface CompactionStatus {
dataSource: string;
scheduleStatus: string;
- bytesAwaitingCompaction: number;
- bytesCompacted: number;
- bytesSkipped: number;
- segmentCountAwaitingCompaction: number;
- segmentCountCompacted: number;
- segmentCountSkipped: number;
- intervalCountAwaitingCompaction: number;
- intervalCountCompacted: number;
- intervalCountSkipped: number;
+ bytesAwaitingCompaction: NumberLike;
+ bytesCompacted: NumberLike;
+ bytesSkipped: NumberLike;
+ segmentCountAwaitingCompaction: NumberLike;
+ segmentCountCompacted: NumberLike;
+ segmentCountSkipped: NumberLike;
+ intervalCountAwaitingCompaction: NumberLike;
+ intervalCountCompacted: NumberLike;
+ intervalCountSkipped: NumberLike;
}
export function zeroCompactionStatus(compactionStatus: CompactionStatus):
boolean {
diff --git a/web-console/src/druid-models/stages/stages.ts
b/web-console/src/druid-models/stages/stages.ts
index c1e2e18096d..9c3a6a3e700 100644
--- a/web-console/src/druid-models/stages/stages.ts
+++ b/web-console/src/druid-models/stages/stages.ts
@@ -19,7 +19,16 @@
import { max, sum } from 'd3-array';
import { AutoForm } from '../../components';
-import { countBy, deleteKeys, filterMap, groupByAsMap, oneOf, zeroDivide }
from '../../utils';
+import type { NumberLike } from '../../utils';
+import {
+ countBy,
+ deleteKeys,
+ filterMap,
+ groupByAsMap,
+ isNumberLike,
+ oneOf,
+ zeroDivide,
+} from '../../utils';
import type { InputFormat } from '../input-format/input-format';
import type { InputSource } from '../input-source/input-source';
@@ -31,8 +40,8 @@ const READING_INPUT_WITH_SHUFFLE_WEIGHT = 1 - SHUFFLE_WEIGHT;
export type InOut = 'in' | 'out';
-function simpleSum(xs: number[]) {
- return sum(xs);
+function simpleSum(xs: NumberLike[]): number {
+ return sum(xs, x => Number(x));
}
function aggregateThings<T>(
@@ -155,7 +164,7 @@ export type ChannelCounterName = `input${number}` |
'output' | 'shuffle';
export type CounterName = keyof StageWorkerCounter;
function tallyWarningCount(warningCounter: WarningCounter): number {
- return sum(Object.values(warningCounter), v => (typeof v === 'number' ? v :
0));
+ return sum(Object.values(warningCounter), v => (isNumberLike(v) ? Number(v)
: 0));
}
function sumByKey(objs: Record<string, number>[]): Record<string, number> {
@@ -172,17 +181,17 @@ function sumByKey(objs: Record<string, number>[]):
Record<string, number> {
export interface ChannelCounter {
type: 'channel';
- rows?: number[];
- bytes?: number[];
- frames?: number[];
- files?: number[];
- totalFiles?: number[];
- loadBytes?: number[];
- loadTime?: number[];
- loadWait?: number[];
- loadFiles?: number[];
- queries?: number[];
- totalQueries?: number[];
+ rows?: NumberLike[];
+ bytes?: NumberLike[];
+ frames?: NumberLike[];
+ files?: NumberLike[];
+ totalFiles?: NumberLike[];
+ loadBytes?: NumberLike[];
+ loadTime?: NumberLike[];
+ loadWait?: NumberLike[];
+ loadFiles?: NumberLike[];
+ queries?: NumberLike[];
+ totalQueries?: NumberLike[];
}
export type ChannelFields =
@@ -236,10 +245,10 @@ export function aggregateSortProgressCounters(
export interface SegmentGenerationProgressCounter {
type: 'segmentGenerationProgress';
- rowsProcessed: number;
- rowsPersisted: number;
- rowsMerged: number;
- rowsPushed: number;
+ rowsProcessed: NumberLike;
+ rowsPersisted: NumberLike;
+ rowsMerged: NumberLike;
+ rowsPushed: NumberLike;
}
export type SegmentGenerationProgressFields =
@@ -298,11 +307,22 @@ export interface CpusCounter {
export interface CpuCounter {
type: 'cpu';
- cpu: number;
- wall: number;
+ cpu: NumberLike;
+ wall: NumberLike;
}
export interface StorageCounter {
+ type: 'storage';
+ localBytesMax?: NumberLike;
+ localBytesReserved: NumberLike;
+ localFilesWritten: NumberLike;
+ localBytesWritten: NumberLike;
+ durableFileCount: NumberLike;
+ durableBytesWritten: NumberLike;
+}
+
+// normalizeStorageCounter() has coerced these, so they are safe to do
arithmetic on
+export interface NormalizedStorageCounter {
type: 'storage';
localBytesMax?: number;
localBytesReserved: number;
@@ -312,7 +332,9 @@ export interface StorageCounter {
durableBytesWritten: number;
}
-function normalizeStorageCounter(s: StorageCounter | undefined):
StorageCounter | undefined {
+function normalizeStorageCounter(
+ s: StorageCounter | undefined,
+): NormalizedStorageCounter | undefined {
if (!s) return;
return {
type: 'storage',
@@ -340,7 +362,7 @@ export interface SimpleWideCounter {
shuffle?: Record<ChannelFields, number>;
segmentGenerationProgress?: SegmentGenerationProgressCounter;
cpu?: CpusCounter;
- storage?: StorageCounter;
+ storage?: NormalizedStorageCounter;
}
function zeroChannelFields(): Record<ChannelFields, number> {
@@ -585,7 +607,7 @@ export class Stages {
return sum(this.getCountersForStage(stage), c => {
const counter = c[counterName];
if (counter?.type !== 'channel') return 0;
- return sum(counter[field] || []);
+ return simpleSum(counter[field] || []);
});
}
@@ -608,7 +630,7 @@ export class Stages {
if (!counters) return 0;
return zeroDivide(
sum(this.getCountersForStage(stage), c => {
- const rowsToSort = c.output ? sum(c.output.rows || []) : 0;
+ const rowsToSort = c.output ? simpleSum(c.output.rows || []) : 0;
const progressDigest = c.sortProgress?.progressDigest || 0;
return Math.floor(rowsToSort * progressDigest);
}),
@@ -622,7 +644,9 @@ export class Stages {
): number {
const { counters } = this;
if (!counters) return 0;
- return sum(this.getCountersForStage(stage), c =>
c.segmentGenerationProgress?.[field] || 0);
+ return sum(this.getCountersForStage(stage), c =>
+ Number(c.segmentGenerationProgress?.[field] || 0),
+ );
}
getChannelCounterNamesForStage(stage: StageDefinition): ChannelCounterName[]
{
@@ -649,17 +673,17 @@ export class Stages {
const c = stageCounters[channel];
newWideCounter[channel] = c
? {
- rows: sum(c.rows || []),
- bytes: sum(c.bytes || []),
- frames: sum(c.frames || []),
- files: sum(c.files || []),
- totalFiles: sum(c.totalFiles || []),
- loadBytes: sum(c.loadBytes || []),
- loadTime: sum(c.loadTime || []),
- loadWait: sum(c.loadWait || []),
- loadFiles: sum(c.loadFiles || []),
- queries: sum(c.queries || []),
- totalQueries: sum(c.totalQueries || []),
+ rows: simpleSum(c.rows || []),
+ bytes: simpleSum(c.bytes || []),
+ frames: simpleSum(c.frames || []),
+ files: simpleSum(c.files || []),
+ totalFiles: simpleSum(c.totalFiles || []),
+ loadBytes: simpleSum(c.loadBytes || []),
+ loadTime: simpleSum(c.loadTime || []),
+ loadWait: simpleSum(c.loadWait || []),
+ loadFiles: simpleSum(c.loadFiles || []),
+ queries: simpleSum(c.queries || []),
+ totalQueries: simpleSum(c.totalQueries || []),
}
: zeroChannelFields();
}
@@ -684,7 +708,7 @@ export class Stages {
// Check if the worker has any wall time recorded
const { cpu } = stageCounters;
if (cpu) {
- const totalWall = sum(CPUS_COUNTER_FIELDS, field => cpu[field]?.wall
|| 0);
+ const totalWall = sum(CPUS_COUNTER_FIELDS, field =>
Number(cpu[field]?.wall || 0));
if (totalWall > 0) return 0;
}
@@ -694,10 +718,10 @@ export class Stages {
const c = stageCounters[channel];
if (!c) return true;
return (
- sum(c.rows || []) === 0 &&
- sum(c.files || []) === 0 &&
- sum(c.bytes || []) === 0 &&
- sum(c.frames || []) === 0
+ simpleSum(c.rows || []) === 0 &&
+ simpleSum(c.files || []) === 0 &&
+ simpleSum(c.bytes || []) === 0 &&
+ simpleSum(c.frames || []) === 0
);
}),
);
@@ -760,15 +784,15 @@ export class Stages {
for (let i = 0; i < n; i++) {
const c = simpleCounters[i][counterName]!; // This must be defined
as we initialized all the counters above
- c.rows += channelCounter.rows?.[i] || 0;
- c.bytes += channelCounter.bytes?.[i] || 0;
- c.frames += channelCounter.frames?.[i] || 0;
- c.files += channelCounter.files?.[i] || 0;
- c.totalFiles += channelCounter.totalFiles?.[i] || 0;
- c.loadBytes += channelCounter.loadBytes?.[i] || 0;
- c.loadTime += channelCounter.loadTime?.[i] || 0;
- c.loadWait += channelCounter.loadWait?.[i] || 0;
- c.loadFiles += channelCounter.loadFiles?.[i] || 0;
+ c.rows += Number(channelCounter.rows?.[i] || 0);
+ c.bytes += Number(channelCounter.bytes?.[i] || 0);
+ c.frames += Number(channelCounter.frames?.[i] || 0);
+ c.files += Number(channelCounter.files?.[i] || 0);
+ c.totalFiles += Number(channelCounter.totalFiles?.[i] || 0);
+ c.loadBytes += Number(channelCounter.loadBytes?.[i] || 0);
+ c.loadTime += Number(channelCounter.loadTime?.[i] || 0);
+ c.loadWait += Number(channelCounter.loadWait?.[i] || 0);
+ c.loadFiles += Number(channelCounter.loadFiles?.[i] || 0);
}
}
}
diff --git
a/web-console/src/druid-models/supervisor-status/supervisor-status.ts
b/web-console/src/druid-models/supervisor-status/supervisor-status.ts
index feb42a0a7f7..f5f1269629d 100644
--- a/web-console/src/druid-models/supervisor-status/supervisor-status.ts
+++ b/web-console/src/druid-models/supervisor-status/supervisor-status.ts
@@ -36,7 +36,7 @@ export interface SupervisorStatus {
publishingTasks: SupervisorStatusTask[];
latestOffsets?: SupervisorOffsetMap;
minimumLag?: SupervisorOffsetMap;
- aggregateLag: number;
+ aggregateLag: NumberLike;
offsetsLastUpdated: string;
suspended: boolean;
healthy: boolean;
@@ -46,15 +46,15 @@ export interface SupervisorStatus {
// autocompact
scheduleStatus: string;
- bytesAwaitingCompaction: number;
- bytesCompacted: number;
- bytesSkipped: number;
- segmentCountAwaitingCompaction: number;
- segmentCountCompacted: number;
- segmentCountSkipped: number;
- intervalCountAwaitingCompaction: number;
- intervalCountCompacted: number;
- intervalCountSkipped: number;
+ bytesAwaitingCompaction: NumberLike;
+ bytesCompacted: NumberLike;
+ bytesSkipped: NumberLike;
+ segmentCountAwaitingCompaction: NumberLike;
+ segmentCountCompacted: NumberLike;
+ segmentCountSkipped: NumberLike;
+ intervalCountAwaitingCompaction: NumberLike;
+ intervalCountCompacted: NumberLike;
+ intervalCountSkipped: NumberLike;
};
}
@@ -93,30 +93,33 @@ export interface RowStats {
}
export interface RowStatsCounter {
- processed: number;
- processedBytes: number;
- processedWithError: number;
- thrownAway: number;
- unparseable: number;
+ processed: NumberLike;
+ processedBytes: NumberLike;
+ processedWithError: NumberLike;
+ thrownAway: NumberLike;
+ unparseable: NumberLike;
}
-function sumRowStatsCounter(rowStats: RowStatsCounter[]): RowStatsCounter {
+// Aggregating coerces away any BigInt, so the result is safe to do arithmetic
on
+export type AggregatedRowStatsCounter = Record<keyof RowStatsCounter, number>;
+
+function sumRowStatsCounter(rowStats: RowStatsCounter[]):
AggregatedRowStatsCounter {
return {
- processed: sum(rowStats, d => d.processed),
- processedBytes: sum(rowStats, d => d.processedBytes),
- processedWithError: sum(rowStats, d => d.processedWithError),
- thrownAway: sum(rowStats, d => d.thrownAway),
- unparseable: sum(rowStats, d => d.unparseable),
+ processed: sum(rowStats, d => Number(d.processed)),
+ processedBytes: sum(rowStats, d => Number(d.processedBytes)),
+ processedWithError: sum(rowStats, d => Number(d.processedWithError)),
+ thrownAway: sum(rowStats, d => Number(d.thrownAway)),
+ unparseable: sum(rowStats, d => Number(d.unparseable)),
};
}
-function maxRowStatsCounter(rowStats: RowStatsCounter[]): RowStatsCounter {
+function maxRowStatsCounter(rowStats: RowStatsCounter[]):
AggregatedRowStatsCounter {
return {
- processed: max(rowStats, d => d.processed) ?? 0,
- processedBytes: max(rowStats, d => d.processedBytes) ?? 0,
- processedWithError: max(rowStats, d => d.processedWithError) ?? 0,
- thrownAway: max(rowStats, d => d.thrownAway) ?? 0,
- unparseable: max(rowStats, d => d.unparseable) ?? 0,
+ processed: max(rowStats, d => Number(d.processed)) ?? 0,
+ processedBytes: max(rowStats, d => Number(d.processedBytes)) ?? 0,
+ processedWithError: max(rowStats, d => Number(d.processedWithError)) ?? 0,
+ thrownAway: max(rowStats, d => Number(d.thrownAway)) ?? 0,
+ unparseable: max(rowStats, d => Number(d.unparseable)) ?? 0,
};
}
@@ -132,7 +135,7 @@ export function getTotalSupervisorStats(
stats: SupervisorStats,
key: RowStatsKey,
activeTaskIds: string[] | undefined,
-): RowStatsCounter {
+): AggregatedRowStatsCounter {
return sumRowStatsCounter(
Object.values(stats).map(s =>
maxRowStatsCounter(
diff --git a/web-console/src/utils/general.tsx
b/web-console/src/utils/general.tsx
index d719df030ce..8565d2afe9b 100644
--- a/web-console/src/utils/general.tsx
+++ b/web-console/src/utils/general.tsx
@@ -41,6 +41,31 @@ export function isNumberLikeNaN(x: NumberLike): boolean {
return isNaN(Number(x));
}
+/**
+ * Query results and API responses are parsed with JSONBig, so any integer
with 16+ digits arrives
+ * as a BigInt. Charting libraries can not do arithmetic on those, so flatten
them back to numbers.
+ *
+ * Only the named (measure) columns are converted. Dimension values are left
alone because a BigInt
+ * above Number.MAX_SAFE_INTEGER would round, merging distinct categories into
one bucket and making
+ * click-to-filter target a value that is not in the data.
+ */
+export function bigIntsToNumbers<T extends Record<string, any>>(
+ rows: T[],
+ measureKeys: string[],
+): T[] {
+ return rows.map(row => {
+ let changed = false;
+ const res: Record<string, any> = { ...row };
+ for (const k of measureKeys) {
+ if (typeof res[k] === 'bigint') {
+ res[k] = Number(res[k]);
+ changed = true;
+ }
+ }
+ return changed ? (res as T) : row;
+ });
+}
+
export function nonEmptyString(s: unknown): s is string {
return typeof s === 'string' && s !== '';
}
diff --git
a/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
b/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
index 5f471fa1e39..5c79521c819 100644
---
a/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
+++
b/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
@@ -188,11 +188,11 @@ exports[`DatasourcesView matches snapshot 1`] = `
{
"Cell": [Function],
"Header": "Availability",
- "accessor": "num_segments",
+ "accessor": [Function],
"className": "padded",
"filterable": false,
+ "id": "num_segments",
"show": true,
- "sortMethod": [Function],
"width": 220,
},
{
diff --git a/web-console/src/views/datasources-view/datasources-view.tsx
b/web-console/src/views/datasources-view/datasources-view.tsx
index 49e0f0e2996..e5196c85bb1 100644
--- a/web-console/src/views/datasources-view/datasources-view.tsx
+++ b/web-console/src/views/datasources-view/datasources-view.tsx
@@ -81,6 +81,7 @@ import {
getDruidErrorMessage,
groupByAsMap,
hasOverlayOpen,
+ isNumberLike,
isNumberLikeNaN,
LocalStorageBackedVisibility,
LocalStorageKeys,
@@ -161,26 +162,27 @@ const formatAvgRowSize = formatInteger;
const formatReplicatedSize = formatBytes;
const formatLeftToBeCompacted = formatBytes;
-function progress(done: number, awaiting: number): number {
- const d = done + awaiting;
- if (!d) return 0;
- return done / d;
+function progress(done: NumberLike, awaiting: NumberLike): number {
+ const doneNumber = Number(done);
+ const total = doneNumber + Number(awaiting);
+ if (!total) return 0;
+ return doneNumber / total;
}
const PERCENT_BRACES = [formatPercent(1)];
interface DatasourceQueryResultRow {
readonly datasource: string;
- readonly num_segments: number;
- readonly num_zero_replica_segments: number;
- readonly num_segments_to_load: number;
- readonly num_segments_to_drop: number;
- readonly minute_aligned_segments: number;
- readonly hour_aligned_segments: number;
- readonly day_aligned_segments: number;
- readonly month_aligned_segments: number;
- readonly year_aligned_segments: number;
- readonly all_granularity_segments: number;
+ readonly num_segments: NumberLike;
+ readonly num_zero_replica_segments: NumberLike;
+ readonly num_segments_to_load: NumberLike;
+ readonly num_segments_to_drop: NumberLike;
+ readonly minute_aligned_segments: NumberLike;
+ readonly hour_aligned_segments: NumberLike;
+ readonly day_aligned_segments: NumberLike;
+ readonly month_aligned_segments: NumberLike;
+ readonly year_aligned_segments: NumberLike;
+ readonly all_granularity_segments: NumberLike;
readonly total_data_size: NumberLike;
readonly replicated_size: NumberLike;
readonly min_segment_rows: NumberLike;
@@ -1247,11 +1249,22 @@ GROUP BY 1, 2`;
show: visibleColumns.shown('Availability'),
filterable: false,
width: 220,
- accessor: 'num_segments',
+ id: 'num_segments',
+ accessor: ({ num_segments, num_segments_to_load }) => {
+ const total = Number(num_segments);
+ if (!total) return 0;
+ return (total - Number(num_segments_to_load)) / total;
+ },
className: 'padded',
- Cell: ({ value: num_segments, original }) => {
- const { datasource, unused, num_segments_to_load,
num_zero_replica_segments, rules } =
- original as Datasource;
+ Cell: ({ original }) => {
+ const {
+ datasource,
+ unused,
+ num_segments,
+ num_segments_to_load,
+ num_zero_replica_segments,
+ rules,
+ } = original as Datasource;
if (unused) {
return (
<span>
@@ -1273,27 +1286,31 @@ GROUP BY 1, 2`;
{pluralIfNeeded(num_segments, 'segment')}
</a>
);
+ if (!isNumberLike(num_segments) ||
!isNumberLike(num_segments_to_load)) {
+ return '-';
+ }
+
+ const numSegments = Number(num_segments);
+ const numSegmentsToLoad = Number(num_segments_to_load);
const percentZeroReplica = (
- Math.floor((num_zero_replica_segments / num_segments) * 1000)
/ 10
+ Math.floor((Number(num_zero_replica_segments) / numSegments) *
1000) / 10
).toFixed(1);
- if (typeof num_segments_to_load !== 'number' || typeof
num_segments !== 'number') {
- return '-';
- } else if (num_segments === 0) {
+ if (numSegments === 0) {
return (
<span>
<span style={{ color: DatasourcesView.EMPTY_COLOR
}}>● </span>
Empty
</span>
);
- } else if (num_segments_to_load === 0) {
+ } else if (numSegmentsToLoad === 0) {
return (
<span>
<span style={{ color:
DatasourcesView.FULLY_AVAILABLE_COLOR }}>
●
</span>
{assemble(
- num_segments !== num_zero_replica_segments
+ numSegments !== Number(num_zero_replica_segments)
? `Fully ${descriptor}`
: undefined,
hasZeroReplicationRule ? `${percentZeroReplica}% deep
storage only` : '',
@@ -1302,9 +1319,9 @@ GROUP BY 1, 2`;
</span>
);
} else {
- const numAvailableSegments = num_segments -
num_segments_to_load;
+ const numAvailableSegments = numSegments - numSegmentsToLoad;
const percentAvailable = (
- Math.floor((numAvailableSegments / num_segments) * 1000) / 10
+ Math.floor((numAvailableSegments / numSegments) * 1000) / 10
).toFixed(1);
return (
<span>
@@ -1319,11 +1336,6 @@ GROUP BY 1, 2`;
);
}
},
- sortMethod: (d1, d2) => {
- const percentAvailable1 = d1.num_available / d1.num_total;
- const percentAvailable2 = d2.num_available / d2.num_total;
- return percentAvailable1 - percentAvailable2 || d1.num_total -
d2.num_total;
- },
},
{
Header: twoLines('Historical', 'load/drop queues'),
@@ -1569,9 +1581,8 @@ GROUP BY 1, 2`;
width: 200,
accessor: ({ compaction }) => {
const status = compaction?.status;
- return status?.bytesCompacted
- ? status.bytesCompacted / (status.bytesAwaitingCompaction +
status.bytesCompacted)
- : 0;
+ if (!status) return 0;
+ return progress(status.bytesCompacted,
status.bytesAwaitingCompaction);
},
filterable: false,
className: 'padded',
diff --git
a/web-console/src/views/explore-view/modules/bar-chart-module/bar-chart-module.tsx
b/web-console/src/views/explore-view/modules/bar-chart-module/bar-chart-module.tsx
index dac24da809e..21d1134d374 100644
---
a/web-console/src/views/explore-view/modules/bar-chart-module/bar-chart-module.tsx
+++
b/web-console/src/views/explore-view/modules/bar-chart-module/bar-chart-module.tsx
@@ -25,7 +25,7 @@ import { useEffect, useMemo, useRef, useState } from 'react';
import { Loader, PortalBubble, type PortalBubbleOpenOn } from
'../../../../components';
import { useQueryManager } from '../../../../hooks';
-import { formatEmpty } from '../../../../utils';
+import { bigIntsToNumbers, formatEmpty } from '../../../../utils';
import { Issue } from '../../components';
import type { ExpressionMeta } from '../../models';
import { ModuleRepository } from '../../module-repository/module-repository';
@@ -151,7 +151,8 @@ ModuleRepository.registerModule<BarChartParameterValues>({
const [sourceDataState, queryManager] = useQueryManager({
query: dataQuery,
processQuery: async (query, signal) => {
- return (await runSqlQuery(query, signal)).toObjectArray();
+ // Only 'met' is coerced, 'dim' is a dimension value that must keep
its exact value
+ return bigIntsToNumbers((await runSqlQuery(query,
signal)).toObjectArray(), ['met']);
},
});
diff --git
a/web-console/src/views/explore-view/modules/multi-axis-chart-module/multi-axis-chart-module.tsx
b/web-console/src/views/explore-view/modules/multi-axis-chart-module/multi-axis-chart-module.tsx
index f945e028345..073065b58fa 100644
---
a/web-console/src/views/explore-view/modules/multi-axis-chart-module/multi-axis-chart-module.tsx
+++
b/web-console/src/views/explore-view/modules/multi-axis-chart-module/multi-axis-chart-module.tsx
@@ -27,6 +27,7 @@ import { useEffect, useMemo, useRef, useState } from 'react';
import { Loader, PortalBubble, type PortalBubbleOpenOn } from
'../../../../components';
import { useQueryManager } from '../../../../hooks';
import {
+ bigIntsToNumbers,
formatInteger,
formatIsoDateRange,
formatNumber,
@@ -114,7 +115,10 @@
ModuleRepository.registerModule<MultiAxisChartParameterValues>({
throw new Error(`Must have a column of type TIMESTAMP for the
multi-axis chart to work`);
}
- return (await runSqlQuery(query, signal)).toObjectArray();
+ return bigIntsToNumbers(
+ (await runSqlQuery(query, signal)).toObjectArray(),
+ measures.map(measure => measure.name),
+ );
},
});
diff --git
a/web-console/src/views/explore-view/modules/pie-chart-module/pie-chart-module.tsx
b/web-console/src/views/explore-view/modules/pie-chart-module/pie-chart-module.tsx
index 81873f0d87a..987ceb96a89 100644
---
a/web-console/src/views/explore-view/modules/pie-chart-module/pie-chart-module.tsx
+++
b/web-console/src/views/explore-view/modules/pie-chart-module/pie-chart-module.tsx
@@ -26,7 +26,7 @@ import { useEffect, useMemo, useRef, useState } from 'react';
import { Loader, PortalBubble, type PortalBubbleOpenOn } from
'../../../../components';
import { useQueryManager } from '../../../../hooks';
import { ColorAssigner } from '../../../../singletons';
-import { formatEmpty, formatNumber } from '../../../../utils';
+import { bigIntsToNumbers, formatEmpty, formatNumber } from
'../../../../utils';
import { Issue } from '../../components';
import type { ExpressionMeta } from '../../models';
import { ModuleRepository } from '../../module-repository/module-repository';
@@ -131,7 +131,7 @@ ModuleRepository.registerModule<PieChartParameterValues>({
query: dataQueries,
processQuery: async ({ mainQuery, limit, splitExpression,
othersPartialQuery }, signal) => {
const result = await runSqlQuery({ query: mainQuery }, signal);
- const data = result.toObjectArray();
+ const data = bigIntsToNumbers(result.toObjectArray(), ['value']);
if (splitExpression && othersPartialQuery) {
const pieValues = result.getColumnByIndex(0)!;
@@ -140,7 +140,11 @@ ModuleRepository.registerModule<PieChartParameterValues>({
const othersResult = await runSqlQuery({
query:
othersPartialQuery.addWhere(splitExpression.notIn(pieValues.slice(0, limit))),
});
- data.push({ name: 'Others', value: othersResult.rows[0][0],
__isOthers: true });
+ data.push({
+ name: 'Others',
+ value: Number(othersResult.rows[0][0]),
+ __isOthers: true,
+ });
}
}
diff --git
a/web-console/src/views/explore-view/modules/time-chart-module/time-chart-module.tsx
b/web-console/src/views/explore-view/modules/time-chart-module/time-chart-module.tsx
index c6c70dccf15..49689bd5572 100644
---
a/web-console/src/views/explore-view/modules/time-chart-module/time-chart-module.tsx
+++
b/web-console/src/views/explore-view/modules/time-chart-module/time-chart-module.tsx
@@ -345,7 +345,7 @@ ModuleRepository.registerModule<TimeChartParameterValues>({
(b): RangeDatum => ({
start: b[TIME_NAME].valueOf(),
end: granularity.shift(b[TIME_NAME], Timezone.UTC, 1).valueOf(),
- measures: measures.map((_, i) => b[getMeasureName(i)]),
+ measures: measures.map((_, i) => Number(b[getMeasureName(i)])),
facet: b[FACET_NAME],
}),
);
diff --git a/web-console/src/views/segments-view/segments-view.tsx
b/web-console/src/views/segments-view/segments-view.tsx
index 7be81490255..90b96a6d621 100644
--- a/web-console/src/views/segments-view/segments-view.tsx
+++ b/web-console/src/views/segments-view/segments-view.tsx
@@ -216,7 +216,7 @@ interface SegmentQueryResultRow {
version: string;
shard_spec: ShardSpec;
partition_num: number;
- size: number;
+ size: NumberLike;
num_rows: NumberLike;
avg_row_size: NumberLike;
num_replicas: number;
diff --git a/web-console/src/views/services-view/services-view.tsx
b/web-console/src/views/services-view/services-view.tsx
index 606c1d58dfa..b02487ce9d1 100644
--- a/web-console/src/views/services-view/services-view.tsx
+++ b/web-console/src/views/services-view/services-view.tsx
@@ -164,8 +164,8 @@ interface ServiceResultRow {
readonly version: string;
readonly build_revision: string;
readonly labels: string | null;
- readonly available_processors: number;
- readonly total_memory: number;
+ readonly available_processors: NumberLike;
+ readonly total_memory: NumberLike;
}
interface CloneStatusInfo {
@@ -174,7 +174,7 @@ interface CloneStatusInfo {
readonly state: string;
readonly segmentLoadsRemaining: number;
readonly segmentDropsRemaining: number;
- readonly bytesToLoad: number;
+ readonly bytesToLoad: NumberLike;
}
interface ServerModeInfo {
@@ -810,7 +810,7 @@ ORDER BY
Aggregated: ({ subRows }) => {
const originalRows = subRows.map(r => r._original);
if (!originalRows.some(r => r.service_type === 'historical'))
return '';
- const totalCurr = sum(originalRows, s => s.curr_size);
+ const totalCurr = sum(originalRows, s => Number(s.curr_size));
return formatBytes(totalCurr);
},
Cell: ({ value, aggregated, original }) => {
@@ -830,7 +830,7 @@ ORDER BY
Aggregated: ({ subRows }) => {
const originalRows = subRows.map(r => r._original);
if (!originalRows.some(r => r.service_type === 'historical'))
return '';
- const totalEffectiveSize = sum(originalRows, s =>
s.effective_size);
+ const totalEffectiveSize = sum(originalRows, s =>
Number(s.effective_size));
return formatBytes(totalEffectiveSize);
},
Cell: ({ value, aggregated, original }) => {
@@ -988,7 +988,7 @@ ORDER BY
Cell: ({ value }) => (value === null ? '' : formatInteger(value)),
Aggregated: ({ subRows }) => {
const originalRows: ServiceResultRow[] = subRows.map(r =>
r._original);
- const totalAvailableProcessors = sum(originalRows, s =>
s.available_processors);
+ const totalAvailableProcessors = sum(originalRows, s =>
Number(s.available_processors));
return totalAvailableProcessors;
},
},
@@ -1005,7 +1005,7 @@ ORDER BY
},
Aggregated: ({ subRows }) => {
const originalRows: ServiceResultRow[] = subRows.map(r =>
r._original);
- const totalMemory = sum(originalRows, s => s.total_memory);
+ const totalMemory = sum(originalRows, s => Number(s.total_memory));
return formatBytes(totalMemory, true);
},
},
diff --git
a/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx
b/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx
index 5a4cb1932f5..888e718a258 100644
---
a/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx
+++
b/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx
@@ -310,11 +310,13 @@ export const ExecutionStagesPane = React.memo(function
ExecutionStagesPane(
<div
key={k}
data-tooltip={`${fieldTitle}\nCPU time:
${formatDurationWithMs(
- v.cpu / 1e6,
+ Number(v.cpu) / 1e6,
)}`}
>
<span
className="cpu-label">{cpusCounterFieldTitle(k)}</span>
- <span
className="cpu-counter">{formatDurationWithMs(v.wall / 1e6)}</span>
+ <span className="cpu-counter">
+ {formatDurationWithMs(Number(v.wall) / 1e6)}
+ </span>
</div>
);
})}
@@ -1036,10 +1038,14 @@ ${title} uncompressed size: ${formatBytesCompact(
return (
<div
key={k}
- data-tooltip={`${fieldTitle}\nCPU time:
${formatDurationWithMs(v.cpu / 1e6)}`}
+ data-tooltip={`${fieldTitle}\nCPU time:
${formatDurationWithMs(
+ Number(v.cpu) / 1e6,
+ )}`}
>
<span className="cpu-label">{fieldTitle}</span>
- <span
className="cpu-counter">{formatDurationWithMs(v.wall / 1e6)}</span>
+ <span className="cpu-counter">
+ {formatDurationWithMs(Number(v.wall) / 1e6)}
+ </span>
</div>
);
})}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
