This is an automated email from the ASF dual-hosted git repository. vogievetsky pushed a commit to branch segment_timeline2 in repository https://gitbox.apache.org/repos/asf/druid.git
commit f08a337e44542b0dcbbf082040ff050346bd05b9 Author: Vadim Ogievetsky <[email protected]> AuthorDate: Wed Oct 30 14:53:11 2024 -0700 auto trimming --- .../src/components/segment-timeline/common.ts | 30 +++-- .../segment-timeline/segment-bar-chart-render.tsx | 147 ++++++++++++++++----- .../segment-timeline/segment-bar-chart.tsx | 91 ++++--------- .../segment-timeline/segment-timeline.tsx | 14 +- web-console/src/utils/general.spec.ts | 7 - web-console/src/utils/general.tsx | 4 - 6 files changed, 165 insertions(+), 128 deletions(-) diff --git a/web-console/src/components/segment-timeline/common.ts b/web-console/src/components/segment-timeline/common.ts index bfb6d84e5f3..966ade312ff 100644 --- a/web-console/src/components/segment-timeline/common.ts +++ b/web-console/src/components/segment-timeline/common.ts @@ -18,34 +18,40 @@ import { sum } from 'd3-array'; -export type SegmentStat = 'count' | 'size' | 'rows'; +import type { Duration } from '../../utils'; + +export type IntervalStat = 'segments' | 'size' | 'rows'; export function aggregateSegmentStats( - xs: readonly Record<SegmentStat, number>[], -): Record<SegmentStat, number> { + xs: readonly Record<IntervalStat, number>[], +): Record<IntervalStat, number> { return { - count: sum(xs, s => s.count), + segments: sum(xs, s => s.segments), size: sum(xs, s => s.size), rows: sum(xs, s => s.rows), }; } -export interface IntervalRow extends Record<SegmentStat, number> { +export interface IntervalRow extends Record<IntervalStat, number> { start: Date; end: Date; - durationSeconds: number; datasource: string; + originalTimeSpan: Duration; +} + +export interface TrimmedIntervalRow extends IntervalRow { + shownSeconds: number; } -export interface SegmentBar extends IntervalRow { - offset: Record<SegmentStat, number>; +export interface IntervalBar extends TrimmedIntervalRow { + offset: Record<IntervalStat, number>; } -export function normalizedSegmentRow(sr: IntervalRow): IntervalRow { +export function normalizeIntervalRow(sr: TrimmedIntervalRow): TrimmedIntervalRow { return { ...sr, - count: sr.count / sr.durationSeconds, - size: sr.size / sr.durationSeconds, - rows: sr.rows / sr.durationSeconds, + segments: sr.segments / sr.shownSeconds, + size: sr.size / sr.shownSeconds, + rows: sr.rows / sr.shownSeconds, }; } diff --git a/web-console/src/components/segment-timeline/segment-bar-chart-render.tsx b/web-console/src/components/segment-timeline/segment-bar-chart-render.tsx index 7864021622d..8753f8bca64 100644 --- a/web-console/src/components/segment-timeline/segment-bar-chart-render.tsx +++ b/web-console/src/components/segment-timeline/segment-bar-chart-render.tsx @@ -17,10 +17,11 @@ */ import type { NonNullDateRange } from '@blueprintjs/datetime'; +import IntervalTree from '@flatten-js/interval-tree'; import classNames from 'classnames'; import { max } from 'd3-array'; import { axisBottom, axisLeft } from 'd3-axis'; -import { scaleLinear, scaleOrdinal, scaleUtc } from 'd3-scale'; +import { scaleLinear, scaleUtc } from 'd3-scale'; import type React from 'react'; import { useMemo, useRef, useState } from 'react'; @@ -29,20 +30,34 @@ import { capitalizeFirst, clamp, day, + Duration, formatByteRate, formatBytes, formatInteger, formatNumber, + groupBy, + hashJoaat, + prettyFormatIsoDate, TZ_UTC, } from '../../utils'; import type { Margin, Stage } from '../../utils/stage'; import { ChartAxis } from './chart-axis'; -import type { SegmentBar, SegmentStat } from './common'; +import type { IntervalBar, IntervalRow, IntervalStat, TrimmedIntervalRow } from './common'; +import { aggregateSegmentStats, normalizeIntervalRow } from './common'; import './segment-bar-chart-render.scss'; const CHART_MARGIN: Margin = { top: 40, right: 5, bottom: 20, left: 80 }; +const MIN_BAR_WIDTH = 2; +const POSSIBLE_GRANULARITIES = [ + new Duration('PT15M'), + new Duration('PT1H'), + new Duration('PT6H'), + new Duration('P1D'), + new Duration('P1M'), + new Duration('P1Y'), +]; const COLORS = [ '#b33040', @@ -63,16 +78,47 @@ const COLORS = [ '#87606c', ]; +const COLORIZER = ({ datasource }: IntervalBar) => { + const hash = hashJoaat(datasource); + return COLORS[hash % COLORS.length]; +}; + function offsetDateRange(dateRange: NonNullDateRange, offset: number): NonNullDateRange { return [new Date(dateRange[0].valueOf() + offset), new Date(dateRange[1].valueOf() + offset)]; } +function stackIntervalRows(trimmedIntervalRows: TrimmedIntervalRow[]): IntervalBar[] { + const sortedIntervalRows = trimmedIntervalRows.sort((a, b) => { + const shownSecondsDiff = b.shownSeconds - a.shownSeconds; + if (shownSecondsDiff) return shownSecondsDiff; + + const timeSpanDiff = + b.originalTimeSpan.getCanonicalLength() - a.originalTimeSpan.getCanonicalLength(); + if (timeSpanDiff) return timeSpanDiff; + + return b.datasource.localeCompare(a.datasource); + }); + + const intervalTree = new IntervalTree(); + return sortedIntervalRows.map(intervalRow => { + intervalRow = normalizeIntervalRow(intervalRow); + const startMs = intervalRow.start.valueOf(); + const endMs = intervalRow.end.valueOf(); + const intervalRowsBelow = intervalTree.search([startMs + 1, startMs + 2]) as IntervalRow[]; + intervalTree.insert([startMs, endMs], intervalRow); + return { + ...intervalRow, + offset: aggregateSegmentStats(intervalRowsBelow), + }; + }); +} + interface SegmentBarChartRenderProps { stage: Stage; dateRange: NonNullDateRange; changeDateRange(dateRange: NonNullDateRange): void; - shownSegmentStat: SegmentStat; - segmentBars: SegmentBar[]; + shownIntervalStat: IntervalStat; + intervalRows: IntervalRow[]; changeActiveDatasource(datasource: string | undefined): void; } @@ -81,13 +127,13 @@ export const SegmentBarChartRender = function SegmentBarChartRender( ) { const { stage, - shownSegmentStat, + shownIntervalStat, dateRange, changeDateRange, - segmentBars, + intervalRows, changeActiveDatasource, } = props; - const [hoverOn, setHoverOn] = useState<SegmentBar>(); + const [hoverOn, setHoverOn] = useState<IntervalBar>(); const [mouseDownAt, setMouseDownAt] = useState< { time: Date; action: 'select' | 'shift' } | undefined >(); @@ -95,6 +141,50 @@ export const SegmentBarChartRender = function SegmentBarChartRender( const [shiftOffset, setShiftOffset] = useState<number | undefined>(); const svgRef = useRef<SVGSVGElement | null>(null); + const trimGranularity = useMemo(() => { + return Duration.pickSmallestGranularityThatFits( + POSSIBLE_GRANULARITIES, + dateRange[1].valueOf() - dateRange[0].valueOf(), + Math.floor(stage.width / MIN_BAR_WIDTH), + ).toString(); + }, [dateRange, stage.width]); + + console.log(trimGranularity); + + const intervalBars = useMemo(() => { + const trimDuration = new Duration(trimGranularity); + const trimmedIntervalRows = intervalRows.map(intervalRow => { + const start = trimDuration.floor(intervalRow.start, TZ_UTC); + const end = trimDuration.ceil(intervalRow.end, TZ_UTC); + return { + ...intervalRow, + start, + end, + shownSeconds: (end.valueOf() - start.valueOf()) / 1000, + }; + }); + + const fullyGroupedSegmentRows = groupBy( + trimmedIntervalRows, + trimmedIntervalRow => + [ + trimmedIntervalRow.start.toISOString(), + trimmedIntervalRow.end.toISOString(), + trimmedIntervalRow.originalTimeSpan, + trimmedIntervalRow.datasource, + ].join('/'), + (trimmedIntervalRows): TrimmedIntervalRow => { + const firstIntervalRow = trimmedIntervalRows[0]; + return { + ...firstIntervalRow, + ...aggregateSegmentStats(trimmedIntervalRows), + }; + }, + ); + + return stackIntervalRows(fullyGroupedSegmentRows); + }, [intervalRows, trimGranularity]); + const innerStage = stage.applyMargin(CHART_MARGIN); const baseTimeScale = scaleUtc().domain(dateRange).range([0, innerStage.width]); @@ -103,19 +193,14 @@ export const SegmentBarChartRender = function SegmentBarChartRender( ? baseTimeScale.copy().domain(offsetDateRange(dateRange, shiftOffset)) : baseTimeScale; - const colorizer = useMemo(() => { - const s = scaleOrdinal().range(COLORS); - return (d: SegmentBar) => s(d.datasource) as string; - }, []); - - const maxStat = max(segmentBars, d => d[shownSegmentStat] + d.offset[shownSegmentStat]); + const maxStat = max(intervalBars, d => d[shownIntervalStat] + d.offset[shownIntervalStat]); const statScale = scaleLinear() .rangeRound([innerStage.height, 0]) .domain([0, maxStat ?? 1]); const formatTick = (n: number) => { - switch (shownSegmentStat) { - case 'count': + switch (shownIntervalStat) { + case 'segments': case 'rows': return formatInteger(n); @@ -125,8 +210,8 @@ export const SegmentBarChartRender = function SegmentBarChartRender( }; const formatTickRate = (n: number) => { - switch (shownSegmentStat) { - case 'count': + switch (shownIntervalStat) { + case 'segments': return formatNumber(n) + ' seg/s'; case 'rows': @@ -185,11 +270,11 @@ export const SegmentBarChartRender = function SegmentBarChartRender( } }); - function segmentBarToRect(segmentBar: SegmentBar) { - const xStart = clamp(timeScale(segmentBar.start), 0, innerStage.width); - const xEnd = clamp(timeScale(segmentBar.end), 0, innerStage.width); - const y0 = statScale(segmentBar.offset[shownSegmentStat]); - const y = statScale(segmentBar[shownSegmentStat] + segmentBar.offset[shownSegmentStat]); + function segmentBarToRect(intervalBar: IntervalBar) { + const xStart = clamp(timeScale(intervalBar.start), 0, innerStage.width); + const xEnd = clamp(timeScale(intervalBar.end), 0, innerStage.width); + const y0 = statScale(intervalBar.offset[shownIntervalStat]); + const y = statScale(intervalBar[shownIntervalStat] + intervalBar.offset[shownIntervalStat]); return { x: xStart, @@ -209,10 +294,10 @@ export const SegmentBarChartRender = function SegmentBarChartRender( ) : hoverOn ? ( <div className="bar-chart-tooltip"> <div>Datasource: {hoverOn.datasource}</div> - <div>Time: {hoverOn.start.toISOString()}</div> + <div>{`Time: ${prettyFormatIsoDate(hoverOn.start)}/${hoverOn.originalTimeSpan}`}</div> <div> - {`${capitalizeFirst(shownSegmentStat)}: ${formatTick( - hoverOn[shownSegmentStat] * hoverOn.durationSeconds, + {`${capitalizeFirst(shownIntervalStat)}: ${formatTick( + hoverOn[shownIntervalStat] * hoverOn.shownSeconds, )}`} </div> </div> @@ -259,21 +344,21 @@ export const SegmentBarChartRender = function SegmentBarChartRender( .tickFormat(e => formatTickRate(e.valueOf()))} /> <g className="bar-group"> - {segmentBars.map((segmentBar, i) => { + {intervalBars.map((intervalBar, i) => { return ( <rect key={i} className="bar-unit" - {...segmentBarToRect(segmentBar)} - style={{ fill: colorizer(segmentBar) }} + {...segmentBarToRect(intervalBar)} + style={{ fill: COLORIZER(intervalBar) }} onClick={ - segmentBar.datasource - ? () => changeActiveDatasource(segmentBar.datasource) + intervalBar.datasource + ? () => changeActiveDatasource(intervalBar.datasource) : undefined } onMouseOver={() => { if (mouseDownAt) return; - setHoverOn(segmentBar); + setHoverOn(intervalBar); }} /> ); diff --git a/web-console/src/components/segment-timeline/segment-bar-chart.tsx b/web-console/src/components/segment-timeline/segment-bar-chart.tsx index 3ba9b16f082..d88a023f883 100644 --- a/web-console/src/components/segment-timeline/segment-bar-chart.tsx +++ b/web-console/src/components/segment-timeline/segment-bar-chart.tsx @@ -18,50 +18,26 @@ import type { NonNullDateRange } from '@blueprintjs/datetime'; import { C, F, N, sql, SqlQuery } from '@druid-toolkit/query'; -import IntervalTree from '@flatten-js/interval-tree'; import { useMemo } from 'react'; import type { Capabilities } from '../../helpers'; import { useQueryManager } from '../../hooks'; import { Api } from '../../singletons'; -import { Duration, filterMap, groupBy, queryDruidSql, TZ_UTC } from '../../utils'; +import { Duration, filterMap, queryDruidSql, TZ_UTC } from '../../utils'; import type { Stage } from '../../utils/stage'; import { Loader } from '../loader/loader'; -import type { IntervalRow, SegmentBar, SegmentStat } from './common'; -import { aggregateSegmentStats, normalizedSegmentRow } from './common'; +import type { IntervalStat } from './common'; import { SegmentBarChartRender } from './segment-bar-chart-render'; import './segment-bar-chart.scss'; -function stackIntervalRows(segmentRows: IntervalRow[]): SegmentBar[] { - const sorted = segmentRows.sort((a, b) => { - const diff = b.durationSeconds - a.durationSeconds; - if (diff) return diff; - if (!a.datasource || !b.datasource) return 0; - return b.datasource.localeCompare(a.datasource); - }); - - const intervalTree = new IntervalTree(); - return sorted.map(segmentRow => { - segmentRow = normalizedSegmentRow(segmentRow); - const startMs = segmentRow.start.valueOf(); - const endMs = segmentRow.end.valueOf(); - const segmentRowsBelow = intervalTree.search([startMs + 1, startMs + 2]) as IntervalRow[]; - intervalTree.insert([startMs, endMs], segmentRow); - return { - ...segmentRow, - offset: aggregateSegmentStats(segmentRowsBelow), - }; - }); -} - interface SegmentBarChartProps { capabilities: Capabilities; stage: Stage; dateRange: NonNullDateRange; changeDateRange(newDateRange: NonNullDateRange): void; - shownSegmentStat: SegmentStat; + shownIntervalStat: IntervalStat; changeActiveDatasource: (datasource: string | undefined) => void; } @@ -71,17 +47,15 @@ export const SegmentBarChart = function SegmentBarChart(props: SegmentBarChartPr dateRange, changeDateRange, stage, - shownSegmentStat, + shownIntervalStat, changeActiveDatasource, } = props; const intervalsQuery = useMemo(() => ({ capabilities, dateRange }), [capabilities, dateRange]); - const [intervalBarsState] = useQueryManager({ + const [intervalRowsState] = useQueryManager({ query: intervalsQuery, processQuery: async ({ capabilities, dateRange }, cancelToken) => { - const trimDuration = new Duration('PT1H'); - let intervalRows: IntervalRow[]; if (capabilities.hasSql()) { const query = SqlQuery.from(N('sys').table('segments')) .changeWhereExpression( @@ -90,25 +64,27 @@ export const SegmentBarChart = function SegmentBarChart(props: SegmentBarChartPr .addSelect(C('start'), { addToGroupBy: 'end' }) .addSelect(C('end'), { addToGroupBy: 'end' }) .addSelect(C('datasource'), { addToGroupBy: 'end' }) - .addSelect(F.count().as('count')) + .addSelect(F.count().as('segments')) .addSelect(F.sum(C('size')).as('size')) .addSelect(F.sum(C('num_rows')).as('rows')); - intervalRows = (await queryDruidSql({ query: query.toString() })).map(sr => { - const start = trimDuration.floor(new Date(sr.start), TZ_UTC); - const end = trimDuration.ceil(new Date(sr.end), TZ_UTC); + return (await queryDruidSql({ query: query.toString() }, cancelToken)).map(sr => { + const start = new Date(sr.start); + const end = new Date(sr.end); + return { ...sr, start, end, - durationSeconds: (end.valueOf() - start.valueOf()) / 1000, + originalTimeSpan: Duration.fromRange(start, end, TZ_UTC), }; }); // This trimming should ideally be pushed into the SQL query but at the time of this writing queries on the sys.* tables do not allow substring } else { const datasources: string[] = ( await Api.instance.get(`/druid/coordinator/v1/datasources`, { cancelToken }) ).data; - intervalRows = ( + + return ( await Promise.all( datasources.map(async datasource => { const intervalMap = ( @@ -121,17 +97,16 @@ export const SegmentBarChart = function SegmentBarChart(props: SegmentBarChartPr ).data; return filterMap(Object.entries(intervalMap), ([interval, v]) => { - // ToDo: Filter on start end const [startStr, endStr] = interval.split('/'); - const start = trimDuration.floor(new Date(startStr), TZ_UTC); - const end = trimDuration.ceil(new Date(endStr), TZ_UTC); + const start = new Date(startStr); + const end = new Date(endStr); + // ToDo: Filter on start end const { count, size, rows } = v as any; return { start, end, - durationSeconds: (end.valueOf() - start.valueOf()) / 1000, datasource, - count, + segments: count, size, rows, }; @@ -140,49 +115,31 @@ export const SegmentBarChart = function SegmentBarChart(props: SegmentBarChartPr ) ).flat(); } - - const fullyGroupedSegmentRows = groupBy( - intervalRows, - intervalRow => - [ - intervalRow.start.toISOString(), - intervalRow.end.toISOString(), - intervalRow.datasource || '', - ].join('/'), - (intervalRows): IntervalRow => { - return { - ...intervalRows[0], - ...aggregateSegmentStats(intervalRows), - }; - }, - ); - - return stackIntervalRows(fullyGroupedSegmentRows); }, }); - if (intervalBarsState.loading) { + if (intervalRowsState.loading) { return <Loader />; } - if (intervalBarsState.error) { + if (intervalRowsState.error) { return ( <div className="empty-placeholder"> - <span className="no-data-text">{`Error when loading data: ${intervalBarsState.getErrorMessage()}`}</span> + <span className="no-data-text">{`Error when loading data: ${intervalRowsState.getErrorMessage()}`}</span> </div> ); } - const segmentBars = intervalBarsState.data; - if (!segmentBars) return null; + const intervalRows = intervalRowsState.data; + if (!intervalRows) return null; return ( <SegmentBarChartRender stage={stage} dateRange={dateRange} changeDateRange={changeDateRange} - shownSegmentStat={shownSegmentStat} - segmentBars={segmentBars} + shownIntervalStat={shownIntervalStat} + intervalRows={intervalRows} changeActiveDatasource={changeActiveDatasource} /> ); diff --git a/web-console/src/components/segment-timeline/segment-timeline.tsx b/web-console/src/components/segment-timeline/segment-timeline.tsx index db6653c912b..5344916c95e 100644 --- a/web-console/src/components/segment-timeline/segment-timeline.tsx +++ b/web-console/src/components/segment-timeline/segment-timeline.tsx @@ -37,7 +37,7 @@ import { import { Stage } from '../../utils/stage'; import { SplitterLayout } from '../splitter-layout/splitter-layout'; -import type { SegmentStat } from './common'; +import type { IntervalStat } from './common'; import { SegmentBarChart } from './segment-bar-chart'; import './segment-timeline.scss'; @@ -56,7 +56,7 @@ function getDefaultDateRange(): NonNullDateRange { export const SegmentTimeline = function SegmentTimeline(props: SegmentTimelineProps) { const { capabilities } = props; const [stage, setStage] = useState<Stage | undefined>(); - const [activeSegmentStat, setActiveSegmentStat] = useState<SegmentStat>('size'); + const [activeSegmentStat, setActiveSegmentStat] = useState<IntervalStat>('size'); const [activeDatasource, setActiveDatasource] = useState<string | undefined>(); const [dateRange, setDateRange] = useState<NonNullDateRange>(getDefaultDateRange); @@ -127,7 +127,7 @@ export const SegmentTimeline = function SegmentTimeline(props: SegmentTimelinePr stage={stage} dateRange={dateRange} changeDateRange={setDateRange} - shownSegmentStat={activeSegmentStat} + shownIntervalStat={activeSegmentStat} changeActiveDatasource={(datasource: string | undefined) => setActiveDatasource(activeDatasource ? undefined : datasource) } @@ -139,7 +139,7 @@ export const SegmentTimeline = function SegmentTimeline(props: SegmentTimelinePr <FormGroup label="Show"> <SegmentedControl value={activeSegmentStat} - onValueChange={s => setActiveSegmentStat(s as SegmentStat)} + onValueChange={s => setActiveSegmentStat(s as IntervalStat)} fill options={[ { @@ -147,12 +147,12 @@ export const SegmentTimeline = function SegmentTimeline(props: SegmentTimelinePr value: 'size', }, { - label: 'Rows', + label: 'Num. rows', value: 'rows', }, { - label: 'Count', - value: 'count', + label: 'Num. segments', + value: 'segments', }, ]} /> diff --git a/web-console/src/utils/general.spec.ts b/web-console/src/utils/general.spec.ts index 4b97cb19acb..e7ad92e7ab7 100644 --- a/web-console/src/utils/general.spec.ts +++ b/web-console/src/utils/general.spec.ts @@ -29,7 +29,6 @@ import { hashJoaat, moveElement, moveToIndex, - objectHash, offsetToRowColumn, parseCsvLine, swapElements, @@ -178,12 +177,6 @@ describe('general', () => { }); }); - describe('objectHash', () => { - it('works', () => { - expect(objectHash({ hello: 'world1' })).toEqual('cc14ad13'); - }); - }); - describe('offsetToRowColumn', () => { it('works', () => { const str = 'Hello\nThis is a test\nstring.'; diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx index 0bc959b6045..9e2e5e7f525 100644 --- a/web-console/src/utils/general.tsx +++ b/web-console/src/utils/general.tsx @@ -620,10 +620,6 @@ export function hashJoaat(str: string): number { return (hash & 4294967295) >>> 0; } -export function objectHash(obj: any): string { - return hashJoaat(JSONBig.stringify(obj)).toString(16).padStart(8); -} - export function hasPopoverOpen(): boolean { return Boolean(document.querySelector(`${Classes.PORTAL} ${Classes.OVERLAY} ${Classes.POPOVER}`)); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
