This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 39ada8b9ade Web console: surface more info on the supervisor view
(#16318)
39ada8b9ade is described below
commit 39ada8b9adecc6beee635a5fd7ead27472904d3b
Author: Vadim Ogievetsky <[email protected]>
AuthorDate: Thu May 2 08:50:27 2024 -0700
Web console: surface more info on the supervisor view (#16318)
* add rate and stats
* better tabs
* detail
* add recent errors
* update tests
* don't let people hide the actions column because why
* don't sort on actions
* better way to agg
* add timeouts
* show error only once
* fix tests and Explain showing up
* only consider active tasks
* refresh
* fix tests
* better formatting
---
.../src/components/header-bar/header-bar.scss | 5 +
.../table-column-selector.spec.tsx | 2 +-
.../table-column-selector.tsx | 43 +-
.../__snapshots__/timed-button.spec.tsx.snap | 2 +-
.../src/components/timed-button/timed-button.tsx | 4 +-
.../compaction-history-dialog.tsx | 4 +-
.../kill-datasource-dialog.tsx | 7 +-
.../supervisor-table-action-dialog.spec.tsx.snap | 4 +-
.../supervisor-statistics-table.spec.tsx.snap | 57 ++-
.../supervisor-statistics-table.spec.tsx | 6 +-
.../supervisor-statistics-table.tsx | 91 +++--
.../supervisor-table-action-dialog.tsx | 14 +-
.../task-table-action-dialog.spec.tsx.snap | 22 +-
.../task-table-action-dialog.tsx | 40 +-
.../supervisor-status/supervisor-status.ts | 85 +++-
web-console/src/utils/druid-query.ts | 9 +-
web-console/src/utils/general.tsx | 12 +
.../src/utils/local-storage-backed-visibility.tsx | 4 +-
web-console/src/utils/table-helpers.ts | 19 +
.../__snapshots__/datasources-view.spec.tsx.snap | 3 +-
.../views/datasources-view/datasources-view.tsx | 10 +-
.../src/views/load-data-view/info-messages.tsx | 6 +-
.../src/views/load-data-view/load-data-view.tsx | 10 +-
.../__snapshots__/lookups-view.spec.tsx.snap | 3 +-
.../src/views/lookups-view/lookups-view.tsx | 3 +-
.../__snapshots__/segments-view.spec.tsx.snap | 2 +-
.../src/views/segments-view/segments-view.tsx | 46 +--
.../__snapshots__/services-view.spec.tsx.snap | 2 +-
.../src/views/services-view/services-view.tsx | 5 +-
.../__snapshots__/supervisors-view.spec.tsx.snap | 148 ++++++-
.../views/supervisors-view/supervisors-view.scss | 13 +
.../views/supervisors-view/supervisors-view.tsx | 451 ++++++++++++++++-----
.../__snapshots__/tasks-view.spec.tsx.snap | 3 +-
web-console/src/views/tasks-view/tasks-view.tsx | 9 +-
.../max-tasks-button/max-tasks-button.tsx | 11 +-
.../src/views/workbench-view/workbench-view.tsx | 18 +-
36 files changed, 853 insertions(+), 320 deletions(-)
diff --git a/web-console/src/components/header-bar/header-bar.scss
b/web-console/src/components/header-bar/header-bar.scss
index 062768a22c4..752cc9bf316 100644
--- a/web-console/src/components/header-bar/header-bar.scss
+++ b/web-console/src/components/header-bar/header-bar.scss
@@ -89,4 +89,9 @@
}
}
}
+
+ .#{$bp-ns}-navbar-group.#{$bp-ns}-align-right {
+ position: absolute;
+ right: 15px;
+ }
}
diff --git
a/web-console/src/components/table-column-selector/table-column-selector.spec.tsx
b/web-console/src/components/table-column-selector/table-column-selector.spec.tsx
index e45fd590e63..e04377c9c5b 100644
---
a/web-console/src/components/table-column-selector/table-column-selector.spec.tsx
+++
b/web-console/src/components/table-column-selector/table-column-selector.spec.tsx
@@ -25,7 +25,7 @@ describe('TableColumnSelector', () => {
it('matches snapshot', () => {
const tableColumn = (
<TableColumnSelector
- columns={['a', 'b', 'c']}
+ columns={['a', 'b', { text: 'c', label: 'c-label' }]}
onChange={() => {}}
tableColumnsHidden={['b']}
/>
diff --git
a/web-console/src/components/table-column-selector/table-column-selector.tsx
b/web-console/src/components/table-column-selector/table-column-selector.tsx
index 2a0c2b5a476..d838e98e04d 100644
--- a/web-console/src/components/table-column-selector/table-column-selector.tsx
+++ b/web-console/src/components/table-column-selector/table-column-selector.tsx
@@ -25,9 +25,15 @@ import { MenuCheckbox } from
'../menu-checkbox/menu-checkbox';
import './table-column-selector.scss';
+export type TableColumnSelectorColumn = string | { text: string; label: string
};
+
+function getColumnName(c: TableColumnSelectorColumn) {
+ return typeof c === 'string' ? c : c.text;
+}
+
interface TableColumnSelectorProps {
- columns: string[];
- onChange: (column: string) => void;
+ columns: TableColumnSelectorColumn[];
+ onChange: (columnName: string) => void;
onClose?: (added: number) => void;
tableColumnsHidden: string[];
}
@@ -38,23 +44,28 @@ export const TableColumnSelector = React.memo(function
TableColumnSelector(
const { columns, onChange, onClose, tableColumnsHidden } = props;
const [added, setAdded] = useState(0);
- const isColumnShown = (column: string) =>
!tableColumnsHidden.includes(column);
+ const isColumnShown = (column: TableColumnSelectorColumn) =>
+ !tableColumnsHidden.includes(getColumnName(column));
const checkboxes = (
<Menu className="table-column-selector-menu">
- {columns.map(column => (
- <MenuCheckbox
- text={column}
- key={column}
- checked={isColumnShown(column)}
- onChange={() => {
- if (!isColumnShown(column)) {
- setAdded(added + 1);
- }
- onChange(column);
- }}
- />
- ))}
+ {columns.map(column => {
+ const columnName = getColumnName(column);
+ return (
+ <MenuCheckbox
+ text={columnName}
+ label={typeof column === 'string' ? undefined : column.label}
+ key={columnName}
+ checked={isColumnShown(column)}
+ onChange={() => {
+ if (!isColumnShown(column)) {
+ setAdded(added + 1);
+ }
+ onChange(columnName);
+ }}
+ />
+ );
+ })}
</Menu>
);
diff --git
a/web-console/src/components/timed-button/__snapshots__/timed-button.spec.tsx.snap
b/web-console/src/components/timed-button/__snapshots__/timed-button.spec.tsx.snap
index b030fdb304b..52fbee10242 100644
---
a/web-console/src/components/timed-button/__snapshots__/timed-button.spec.tsx.snap
+++
b/web-console/src/components/timed-button/__snapshots__/timed-button.spec.tsx.snap
@@ -18,7 +18,7 @@ exports[`TimedButton matches snapshot 1`] = `
<Blueprint4.MenuItem
active={false}
disabled={false}
- icon="selection"
+ icon="tick-circle"
multiline={false}
onClick={[Function]}
popoverProps={{}}
diff --git a/web-console/src/components/timed-button/timed-button.tsx
b/web-console/src/components/timed-button/timed-button.tsx
index 0b339a8d0e5..cb275370b27 100644
--- a/web-console/src/components/timed-button/timed-button.tsx
+++ b/web-console/src/components/timed-button/timed-button.tsx
@@ -25,7 +25,7 @@ import React, { useState } from 'react';
import { useInterval } from '../../hooks';
import type { LocalStorageKeys } from '../../utils';
-import { isInBackground, localStorageGet, localStorageSet } from '../../utils';
+import { checkedCircleIcon, isInBackground, localStorageGet, localStorageSet }
from '../../utils';
export interface DelayLabel {
label: string;
@@ -84,7 +84,7 @@ export const TimedButton = React.memo(function
TimedButton(props: TimedButtonPro
{delays.map(({ label, delay }, i) => (
<MenuItem
key={i}
- icon={selectedDelay === delay ? IconNames.SELECTION :
IconNames.CIRCLE}
+ icon={checkedCircleIcon(selectedDelay === delay)}
text={label}
onClick={() => handleSelection(delay)}
/>
diff --git
a/web-console/src/dialogs/compaction-history-dialog/compaction-history-dialog.tsx
b/web-console/src/dialogs/compaction-history-dialog/compaction-history-dialog.tsx
index 4cdc916ee74..cb886d0483d 100644
---
a/web-console/src/dialogs/compaction-history-dialog/compaction-history-dialog.tsx
+++
b/web-console/src/dialogs/compaction-history-dialog/compaction-history-dialog.tsx
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import { Button, Callout, Classes, Code, Dialog, Tab, Tabs } from
'@blueprintjs/core';
+import { Button, Callout, Classes, Dialog, Tab, Tabs, Tag } from
'@blueprintjs/core';
import * as JSONBig from 'json-bigint-native';
import React, { useState } from 'react';
@@ -117,7 +117,7 @@ export const CompactionHistoryDialog = React.memo(function
CompactionHistoryDial
</Tabs>
) : (
<div>
- There is no compaction history for <Code>{datasource}</Code>.
+ There is no compaction history for <Tag
minimal>{datasource}</Tag>.
</div>
)
) : historyState.loading ? (
diff --git
a/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx
b/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx
index f95a5a5d3b8..dba85268d00 100644
--- a/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx
+++ b/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import { Code, Intent } from '@blueprintjs/core';
+import { Intent, Tag } from '@blueprintjs/core';
import React, { useState } from 'react';
import { FormGroupWithInfo, PopoverText } from '../../components';
@@ -74,13 +74,14 @@ export const KillDatasourceDialog = function
KillDatasourceDialog(
warningChecks={[
<>
I understand that this operation will delete all metadata about the
unused segments of{' '}
- <Code>{datasource}</Code> and removes them from deep storage.
+ <Tag minimal>{datasource}</Tag> and removes them from deep storage.
</>,
'I understand that this operation cannot be undone.',
]}
>
<p>
- Are you sure you want to permanently delete unused segments in
<Code>{datasource}</Code>?
+ Are you sure you want to permanently delete unused segments in{' '}
+ <Tag minimal>{datasource}</Tag>?
</p>
<p>This action is not reversible and the data deleted will be lost.</p>
<FormGroupWithInfo
diff --git
a/web-console/src/dialogs/supervisor-table-action-dialog/__snapshots__/supervisor-table-action-dialog.spec.tsx.snap
b/web-console/src/dialogs/supervisor-table-action-dialog/__snapshots__/supervisor-table-action-dialog.spec.tsx.snap
index 7aaa8b1afa2..68cef1ace6e 100755
---
a/web-console/src/dialogs/supervisor-table-action-dialog/__snapshots__/supervisor-table-action-dialog.spec.tsx.snap
+++
b/web-console/src/dialogs/supervisor-table-action-dialog/__snapshots__/supervisor-table-action-dialog.spec.tsx.snap
@@ -116,7 +116,7 @@ exports[`SupervisorTableActionDialog matches snapshot 1`] =
`
<span
class="bp4-button-text"
>
- Statistics
+ Task stats
</span>
</button>
<button
@@ -144,7 +144,7 @@ exports[`SupervisorTableActionDialog matches snapshot 1`] =
`
<span
class="bp4-button-text"
>
- Payload
+ Spec
</span>
</button>
<button
diff --git
a/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/__snapshots__/supervisor-statistics-table.spec.tsx.snap
b/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/__snapshots__/supervisor-statistics-table.spec.tsx.snap
index 7ef6d412fb1..9f61d78e1ec 100644
---
a/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/__snapshots__/supervisor-statistics-table.spec.tsx.snap
+++
b/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/__snapshots__/supervisor-statistics-table.spec.tsx.snap
@@ -78,17 +78,22 @@ exports[`SupervisorStatisticsTable matches snapshot on
error 1`] = `
}
columns={
[
+ {
+ "Header": "Group ID",
+ "accessor": "groupId",
+ "className": "padded",
+ "width": 100,
+ },
{
"Header": "Task ID",
- "accessor": [Function],
+ "accessor": "taskId",
"className": "padded",
- "id": "task_id",
"width": 400,
},
{
"Cell": [Function],
"Header": "Totals",
- "accessor": [Function],
+ "accessor": "rowStats.totals.buildSegments",
"className": "padded",
"id": "total",
"width": 200,
@@ -256,17 +261,22 @@ exports[`SupervisorStatisticsTable matches snapshot on
init 1`] = `
}
columns={
[
+ {
+ "Header": "Group ID",
+ "accessor": "groupId",
+ "className": "padded",
+ "width": 100,
+ },
{
"Header": "Task ID",
- "accessor": [Function],
+ "accessor": "taskId",
"className": "padded",
- "id": "task_id",
"width": 400,
},
{
"Cell": [Function],
"Header": "Totals",
- "accessor": [Function],
+ "accessor": "rowStats.totals.buildSegments",
"className": "padded",
"id": "total",
"width": 200,
@@ -460,17 +470,22 @@ exports[`SupervisorStatisticsTable matches snapshot on no
data 1`] = `
}
columns={
[
+ {
+ "Header": "Group ID",
+ "accessor": "groupId",
+ "className": "padded",
+ "width": 100,
+ },
{
"Header": "Task ID",
- "accessor": [Function],
+ "accessor": "taskId",
"className": "padded",
- "id": "task_id",
"width": 400,
},
{
"Cell": [Function],
"Header": "Totals",
- "accessor": [Function],
+ "accessor": "rowStats.totals.buildSegments",
"className": "padded",
"id": "total",
"width": 200,
@@ -638,17 +653,22 @@ exports[`SupervisorStatisticsTable matches snapshot on
some data 1`] = `
}
columns={
[
+ {
+ "Header": "Group ID",
+ "accessor": "groupId",
+ "className": "padded",
+ "width": 100,
+ },
{
"Header": "Task ID",
- "accessor": [Function],
+ "accessor": "taskId",
"className": "padded",
- "id": "task_id",
"width": 400,
},
{
"Cell": [Function],
"Header": "Totals",
- "accessor": [Function],
+ "accessor": "rowStats.totals.buildSegments",
"className": "padded",
"id": "total",
"width": 200,
@@ -656,7 +676,7 @@ exports[`SupervisorStatisticsTable matches snapshot on some
data 1`] = `
{
"Cell": [Function],
"Header": "1m",
- "accessor": [Function],
+ "accessor": "rowStats.movingAverages.buildSegments.1m",
"className": "padded",
"id": "1m",
"width": 200,
@@ -664,7 +684,7 @@ exports[`SupervisorStatisticsTable matches snapshot on some
data 1`] = `
{
"Cell": [Function],
"Header": "5m",
- "accessor": [Function],
+ "accessor": "rowStats.movingAverages.buildSegments.5m",
"className": "padded",
"id": "5m",
"width": 200,
@@ -672,7 +692,7 @@ exports[`SupervisorStatisticsTable matches snapshot on some
data 1`] = `
{
"Cell": [Function],
"Header": "15m",
- "accessor": [Function],
+ "accessor": "rowStats.movingAverages.buildSegments.15m",
"className": "padded",
"id": "15m",
"width": 200,
@@ -682,23 +702,27 @@ exports[`SupervisorStatisticsTable matches snapshot on
some data 1`] = `
data={
[
{
- "summary": {
+ "groupId": "0",
+ "rowStats": {
"movingAverages": {
"buildSegments": {
"15m": {
"processed": 5.544749689510444,
+ "processedBytes": 20,
"processedWithError": 0,
"thrownAway": 0,
"unparseable": 0,
},
"1m": {
"processed": 4.593670088770785,
+ "processedBytes": 30,
"processedWithError": 0,
"thrownAway": 0,
"unparseable": 0,
},
"5m": {
"processed": 3.5455993615040584,
+ "processedBytes": 10,
"processedWithError": 0,
"thrownAway": 0,
"unparseable": 0,
@@ -708,6 +732,7 @@ exports[`SupervisorStatisticsTable matches snapshot on some
data 1`] = `
"totals": {
"buildSegments": {
"processed": 7516,
+ "processedBytes": 60,
"processedWithError": 0,
"thrownAway": 0,
"unparseable": 0,
diff --git
a/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/supervisor-statistics-table.spec.tsx
b/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/supervisor-statistics-table.spec.tsx
index 3ea907dc0cf..922342550f7 100644
---
a/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/supervisor-statistics-table.spec.tsx
+++
b/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/supervisor-statistics-table.spec.tsx
@@ -28,7 +28,7 @@ import {
} from './supervisor-statistics-table';
let supervisorStatisticsState: QueryState<SupervisorStatisticsTableRow[]> =
QueryState.INIT;
-jest.mock('../../../hooks', () => {
+jest.mock('../../../hooks/use-query-manager', () => {
return {
useQueryManager: () => [supervisorStatisticsState],
};
@@ -72,18 +72,21 @@ describe('SupervisorStatisticsTable', () => {
buildSegments: {
'5m': {
processed: 3.5455993615040584,
+ processedBytes: 10,
unparseable: 0,
thrownAway: 0,
processedWithError: 0,
},
'15m': {
processed: 5.544749689510444,
+ processedBytes: 20,
unparseable: 0,
thrownAway: 0,
processedWithError: 0,
},
'1m': {
processed: 4.593670088770785,
+ processedBytes: 30,
unparseable: 0,
thrownAway: 0,
processedWithError: 0,
@@ -93,6 +96,7 @@ describe('SupervisorStatisticsTable', () => {
totals: {
buildSegments: {
processed: 7516,
+ processedBytes: 60,
processedWithError: 0,
thrownAway: 0,
unparseable: 0,
diff --git
a/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/supervisor-statistics-table.tsx
b/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/supervisor-statistics-table.tsx
index 09749f85253..49525dfb870 100644
---
a/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/supervisor-statistics-table.tsx
+++
b/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/supervisor-statistics-table.tsx
@@ -22,35 +22,26 @@ import type { CellInfo, Column } from 'react-table';
import ReactTable from 'react-table';
import { Loader } from '../../../components/loader/loader';
-import { useQueryManager } from '../../../hooks';
+import type { RowStats, RowStatsCounter, SupervisorStats } from
'../../../druid-models';
+import { useInterval, useQueryManager } from '../../../hooks';
import { SMALL_TABLE_PAGE_SIZE, SMALL_TABLE_PAGE_SIZE_OPTIONS } from
'../../../react-table';
import { Api, UrlBaser } from '../../../singletons';
-import { deepGet } from '../../../utils';
+import { deepGet, formatByteRate, formatBytes, formatInteger, formatRate }
from '../../../utils';
import './supervisor-statistics-table.scss';
-export interface TaskSummary {
- totals: Record<string, StatsEntry>;
- movingAverages: Record<string, Record<string, StatsEntry>>;
-}
-
-export interface StatsEntry {
- processed?: number;
- processedWithError?: number;
- thrownAway?: number;
- unparseable?: number;
- [key: string]: number | undefined;
-}
-
export interface SupervisorStatisticsTableRow {
+ groupId: string;
taskId: string;
- summary: TaskSummary;
+ rowStats: RowStats;
}
export function normalizeSupervisorStatisticsResults(
- data: Record<string, Record<string, TaskSummary>>,
+ data: SupervisorStats,
): SupervisorStatisticsTableRow[] {
- return Object.values(data).flatMap(v => Object.keys(v).map(k => ({ taskId:
k, summary: v[k] })));
+ return Object.entries(data).flatMap(([groupId, v]) =>
+ Object.entries(v).map(([taskId, rowStats]) => ({ groupId, taskId, rowStats
})),
+ );
}
export interface SupervisorStatisticsTableProps {
@@ -62,34 +53,54 @@ export const SupervisorStatisticsTable =
React.memo(function SupervisorStatistic
props: SupervisorStatisticsTableProps,
) {
const { supervisorId } = props;
- const endpoint =
`/druid/indexer/v1/supervisor/${Api.encodePath(supervisorId)}/stats`;
+ const statsEndpoint =
`/druid/indexer/v1/supervisor/${Api.encodePath(supervisorId)}/stats`;
- const [supervisorStatisticsState] = useQueryManager<null,
SupervisorStatisticsTableRow[]>({
+ const [supervisorStatisticsState, supervisorStatisticsQueryManager] =
useQueryManager<
+ null,
+ SupervisorStatisticsTableRow[]
+ >({
+ initQuery: null,
processQuery: async () => {
- const resp = await Api.instance.get(endpoint);
+ const resp = await Api.instance.get<SupervisorStats>(statsEndpoint);
return normalizeSupervisorStatisticsResults(resp.data);
},
- initQuery: null,
});
- function renderCell(cell: CellInfo) {
- const cellValue = cell.value;
- if (!cellValue) {
- return <div>No data found</div>;
- }
+ useInterval(() => {
+ supervisorStatisticsQueryManager.rerunLastQuery(true);
+ }, 1500);
- return Object.keys(cellValue)
- .sort()
- .map(key => <div key={key}>{`${key}:
${Number(cellValue[key]).toFixed(1)}`}</div>);
+ function renderCounters(cell: CellInfo, isRate: boolean) {
+ const c: RowStatsCounter = cell.value;
+ if (!c) return null;
+
+ const formatNumber = isRate ? formatRate : formatInteger;
+ const formatData = isRate ? formatByteRate : formatBytes;
+ const bytes = c.processedBytes ? ` (${formatData(c.processedBytes)})` : '';
+ return (
+ <div>
+ <div>{`Processed: ${formatNumber(c.processed)}${bytes}`}</div>
+ {Boolean(c.processedWithError) && (
+ <div>Processed with error: {formatNumber(c.processedWithError)}</div>
+ )}
+ {Boolean(c.thrownAway) && <div>Thrown away:
{formatNumber(c.thrownAway)}</div>}
+ {Boolean(c.unparseable) && <div>Unparseable:
{formatNumber(c.unparseable)}</div>}
+ </div>
+ );
}
function renderTable() {
let columns: Column<SupervisorStatisticsTableRow>[] = [
+ {
+ Header: 'Group ID',
+ accessor: 'groupId',
+ className: 'padded',
+ width: 100,
+ },
{
Header: 'Task ID',
- id: 'task_id',
+ accessor: 'taskId',
className: 'padded',
- accessor: d => d.taskId,
width: 400,
},
{
@@ -97,16 +108,14 @@ export const SupervisorStatisticsTable =
React.memo(function SupervisorStatistic
id: 'total',
className: 'padded',
width: 200,
- accessor: d => {
- return deepGet(d, 'summary.totals.buildSegments') as StatsEntry;
- },
- Cell: renderCell,
+ accessor: 'rowStats.totals.buildSegments',
+ Cell: c => renderCounters(c, false),
},
];
const movingAveragesBuildSegments = deepGet(
supervisorStatisticsState.data as any,
- '0.summary.movingAverages.buildSegments',
+ '0.rowStats.movingAverages.buildSegments',
);
if (movingAveragesBuildSegments) {
columns = columns.concat(
@@ -118,10 +127,8 @@ export const SupervisorStatisticsTable =
React.memo(function SupervisorStatistic
id: interval,
className: 'padded',
width: 200,
- accessor: d => {
- return deepGet(d,
`summary.movingAverages.buildSegments.${interval}`);
- },
- Cell: renderCell,
+ accessor: `rowStats.movingAverages.buildSegments.${interval}`,
+ Cell: c => renderCounters(c, true),
};
}),
);
@@ -148,7 +155,7 @@ export const SupervisorStatisticsTable =
React.memo(function SupervisorStatistic
text="View raw"
disabled={supervisorStatisticsState.loading}
minimal
- onClick={() => window.open(UrlBaser.base(endpoint), '_blank')}
+ onClick={() => window.open(UrlBaser.base(statsEndpoint), '_blank')}
/>
</ButtonGroup>
</div>
diff --git
a/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-table-action-dialog.tsx
b/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-table-action-dialog.tsx
index 02d9e3c28b8..5e3d9e50028 100644
---
a/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-table-action-dialog.tsx
+++
b/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-table-action-dialog.tsx
@@ -28,6 +28,8 @@ import { TableActionDialog } from
'../table-action-dialog/table-action-dialog';
import { SupervisorStatisticsTable } from
'./supervisor-statistics-table/supervisor-statistics-table';
+type SupervisorTableActionDialogTab = 'status' | 'stats' | 'spec' | 'history';
+
interface SupervisorTableActionDialogProps {
supervisorId: string;
actions: BasicAction[];
@@ -38,7 +40,7 @@ export const SupervisorTableActionDialog =
React.memo(function SupervisorTableAc
props: SupervisorTableActionDialogProps,
) {
const { supervisorId, actions, onClose } = props;
- const [activeTab, setActiveTab] = useState('status');
+ const [activeTab, setActiveTab] =
useState<SupervisorTableActionDialogTab>('status');
const supervisorTableSideButtonMetadata: SideButtonMetaData[] = [
{
@@ -49,15 +51,15 @@ export const SupervisorTableActionDialog =
React.memo(function SupervisorTableAc
},
{
icon: 'chart',
- text: 'Statistics',
+ text: 'Task stats',
active: activeTab === 'stats',
onClick: () => setActiveTab('stats'),
},
{
icon: 'align-left',
- text: 'Payload',
- active: activeTab === 'payload',
- onClick: () => setActiveTab('payload'),
+ text: 'Spec',
+ active: activeTab === 'spec',
+ onClick: () => setActiveTab('spec'),
},
{
icon: 'history',
@@ -88,7 +90,7 @@ export const SupervisorTableActionDialog =
React.memo(function SupervisorTableAc
downloadFilename={`supervisor-stats-${supervisorId}.json`}
/>
)}
- {activeTab === 'payload' && (
+ {activeTab === 'spec' && (
<ShowJson
endpoint={supervisorEndpointBase}
transform={x => cleanSpec(x, true)}
diff --git
a/web-console/src/dialogs/task-table-action-dialog/__snapshots__/task-table-action-dialog.spec.tsx.snap
b/web-console/src/dialogs/task-table-action-dialog/__snapshots__/task-table-action-dialog.spec.tsx.snap
index 4c0ceed638d..63e1e50a863 100644
---
a/web-console/src/dialogs/task-table-action-dialog/__snapshots__/task-table-action-dialog.spec.tsx.snap
+++
b/web-console/src/dialogs/task-table-action-dialog/__snapshots__/task-table-action-dialog.spec.tsx.snap
@@ -97,18 +97,18 @@ exports[`TaskTableActionDialog matches snapshot 1`] = `
>
<span
aria-hidden="true"
- class="bp4-icon bp4-icon-align-left"
- icon="align-left"
+ class="bp4-icon bp4-icon-comparison"
+ icon="comparison"
>
<svg
- data-icon="align-left"
+ data-icon="comparison"
height="20"
role="img"
viewBox="0 0 20 20"
width="20"
>
<path
- d="M1 7h10c.55 0 1-.45 1-1s-.45-1-1-1H1c-.55 0-1 .45-1
1s.45 1 1 1zm0-4h18c.55 0 1-.45 1-1s-.45-1-1-1H1c-.55 0-1 .45-1 1s.45 1 1 1zm14
14H1c-.55 0-1 .45-1 1s.45 1 1 1h14c.55 0 1-.45 1-1s-.45-1-1-1zm4-8H1c-.55 0-1
.45-1 1s.45 1 1 1h18c.55 0 1-.45 1-1s-.45-1-1-1zM1 15h6c.55 0 1-.45
1-1s-.45-1-1-1H1c-.55 0-1 .45-1 1s.45 1 1 1z"
+ d="M6 8H1c-.55 0-1 .45-1 1v2c0 .55.45 1 1 1h5c.55 0 1-.45
1-1V9c0-.55-.45-1-1-1zm13-6h-5c-.55 0-1 .45-1 1v2c0 .55.45 1 1 1h5c.55 0 1-.45
1-1V3c0-.55-.45-1-1-1zm0 3h-5V3h5v2zM6 14H1c-.55 0-1 .45-1 1v2c0 .55.45 1 1
1h5c.55 0 1-.45 1-1v-2c0-.55-.45-1-1-1zM6 2H1c-.55 0-1 .45-1 1v2c0 .55.45 1 1
1h5c.55 0 1-.45 1-1V3c0-.55-.45-1-1-1zm4-2c-.55 0-1 .45-1 1v18c0 .55.45 1 1
1s1-.45 1-1V1c0-.55-.45-1-1-1zm9 14h-5c-.55 0-1 .45-1 1v2c0 .55.45 1 1 1h5c.55
0 1-.45 1-1v-2c0-.55-.45-1 [...]
fill-rule="evenodd"
/>
</svg>
@@ -116,7 +116,7 @@ exports[`TaskTableActionDialog matches snapshot 1`] = `
<span
class="bp4-button-text"
>
- Payload
+ Reports
</span>
</button>
<button
@@ -125,18 +125,18 @@ exports[`TaskTableActionDialog matches snapshot 1`] = `
>
<span
aria-hidden="true"
- class="bp4-icon bp4-icon-comparison"
- icon="comparison"
+ class="bp4-icon bp4-icon-align-left"
+ icon="align-left"
>
<svg
- data-icon="comparison"
+ data-icon="align-left"
height="20"
role="img"
viewBox="0 0 20 20"
width="20"
>
<path
- d="M6 8H1c-.55 0-1 .45-1 1v2c0 .55.45 1 1 1h5c.55 0 1-.45
1-1V9c0-.55-.45-1-1-1zm13-6h-5c-.55 0-1 .45-1 1v2c0 .55.45 1 1 1h5c.55 0 1-.45
1-1V3c0-.55-.45-1-1-1zm0 3h-5V3h5v2zM6 14H1c-.55 0-1 .45-1 1v2c0 .55.45 1 1
1h5c.55 0 1-.45 1-1v-2c0-.55-.45-1-1-1zM6 2H1c-.55 0-1 .45-1 1v2c0 .55.45 1 1
1h5c.55 0 1-.45 1-1V3c0-.55-.45-1-1-1zm4-2c-.55 0-1 .45-1 1v18c0 .55.45 1 1
1s1-.45 1-1V1c0-.55-.45-1-1-1zm9 14h-5c-.55 0-1 .45-1 1v2c0 .55.45 1 1 1h5c.55
0 1-.45 1-1v-2c0-.55-.45-1 [...]
+ d="M1 7h10c.55 0 1-.45 1-1s-.45-1-1-1H1c-.55 0-1 .45-1
1s.45 1 1 1zm0-4h18c.55 0 1-.45 1-1s-.45-1-1-1H1c-.55 0-1 .45-1 1s.45 1 1 1zm14
14H1c-.55 0-1 .45-1 1s.45 1 1 1h14c.55 0 1-.45 1-1s-.45-1-1-1zm4-8H1c-.55 0-1
.45-1 1s.45 1 1 1h18c.55 0 1-.45 1-1s-.45-1-1-1zM1 15h6c.55 0 1-.45
1-1s-.45-1-1-1H1c-.55 0-1 .45-1 1s.45 1 1 1z"
fill-rule="evenodd"
/>
</svg>
@@ -144,7 +144,7 @@ exports[`TaskTableActionDialog matches snapshot 1`] = `
<span
class="bp4-button-text"
>
- Reports
+ Spec
</span>
</button>
<button
@@ -172,7 +172,7 @@ exports[`TaskTableActionDialog matches snapshot 1`] = `
<span
class="bp4-button-text"
>
- Logs
+ Log
</span>
</button>
</div>
diff --git
a/web-console/src/dialogs/task-table-action-dialog/task-table-action-dialog.tsx
b/web-console/src/dialogs/task-table-action-dialog/task-table-action-dialog.tsx
index a0a5dbbf13f..9edc5d996f4 100644
---
a/web-console/src/dialogs/task-table-action-dialog/task-table-action-dialog.tsx
+++
b/web-console/src/dialogs/task-table-action-dialog/task-table-action-dialog.tsx
@@ -25,18 +25,20 @@ import type { BasicAction } from '../../utils/basic-action';
import type { SideButtonMetaData } from
'../table-action-dialog/table-action-dialog';
import { TableActionDialog } from '../table-action-dialog/table-action-dialog';
+type TaskTableActionDialogTab = 'status' | 'report' | 'spec' | 'log';
+
interface TaskTableActionDialogProps {
taskId: string;
actions: BasicAction[];
- onClose: () => void;
status: string;
+ onClose(): void;
}
export const TaskTableActionDialog = React.memo(function TaskTableActionDialog(
props: TaskTableActionDialogProps,
) {
const { taskId, actions, onClose, status } = props;
- const [activeTab, setActiveTab] = useState('status');
+ const [activeTab, setActiveTab] =
useState<TaskTableActionDialogTab>('status');
const taskTableSideButtonMetadata: SideButtonMetaData[] = [
{
@@ -45,21 +47,21 @@ export const TaskTableActionDialog = React.memo(function
TaskTableActionDialog(
active: activeTab === 'status',
onClick: () => setActiveTab('status'),
},
- {
- icon: 'align-left',
- text: 'Payload',
- active: activeTab === 'payload',
- onClick: () => setActiveTab('payload'),
- },
{
icon: 'comparison',
text: 'Reports',
- active: activeTab === 'reports',
- onClick: () => setActiveTab('reports'),
+ active: activeTab === 'report',
+ onClick: () => setActiveTab('report'),
+ },
+ {
+ icon: 'align-left',
+ text: 'Spec',
+ active: activeTab === 'spec',
+ onClick: () => setActiveTab('spec'),
},
{
icon: 'align-justify',
- text: 'Logs',
+ text: 'Log',
active: activeTab === 'log',
onClick: () => setActiveTab('log'),
},
@@ -80,20 +82,20 @@ export const TaskTableActionDialog = React.memo(function
TaskTableActionDialog(
downloadFilename={`task-status-${taskId}.json`}
/>
)}
- {activeTab === 'payload' && (
- <ShowJson
- endpoint={taskEndpointBase}
- transform={x => deepGet(x, 'payload') || x}
- downloadFilename={`task-payload-${taskId}.json`}
- />
- )}
- {activeTab === 'reports' && (
+ {activeTab === 'report' && (
<ShowJson
endpoint={`${taskEndpointBase}/reports`}
transform={x => deepGet(x, 'ingestionStatsAndErrors.payload') || x}
downloadFilename={`task-reports-${taskId}.json`}
/>
)}
+ {activeTab === 'spec' && (
+ <ShowJson
+ endpoint={taskEndpointBase}
+ transform={x => deepGet(x, 'payload') || x}
+ downloadFilename={`task-payload-${taskId}.json`}
+ />
+ )}
{activeTab === 'log' && (
<ShowLog
tail={status === 'RUNNING'}
diff --git
a/web-console/src/druid-models/supervisor-status/supervisor-status.ts
b/web-console/src/druid-models/supervisor-status/supervisor-status.ts
index b25c9f5fb1f..3004cf350e7 100644
--- a/web-console/src/druid-models/supervisor-status/supervisor-status.ts
+++ b/web-console/src/druid-models/supervisor-status/supervisor-status.ts
@@ -16,7 +16,10 @@
* limitations under the License.
*/
+import { max, sum } from 'd3-array';
+
import type { NumberLike } from '../../utils';
+import { deepGet, filterMap } from '../../utils';
export type SupervisorOffsetMap = Record<string, NumberLike>;
@@ -39,16 +42,94 @@ export interface SupervisorStatus {
healthy: boolean;
state: string;
detailedState: string;
- recentErrors: any[];
+ recentErrors: SupervisorError[];
};
}
export interface SupervisorStatusTask {
id: string;
startingOffsets: SupervisorOffsetMap;
- startTime: '2024-04-12T21:35:34.834Z';
+ startTime: string;
remainingSeconds: number;
type: string;
currentOffsets: SupervisorOffsetMap;
lag: SupervisorOffsetMap;
}
+
+export interface SupervisorError {
+ timestamp: string;
+ exceptionClass: string;
+ message: string;
+ streamException: boolean;
+}
+
+export type SupervisorStats = Record<string, Record<string, RowStats>>;
+
+export type RowStatsKey = 'totals' | '1m' | '5m' | '15m';
+
+export interface RowStats {
+ movingAverages: {
+ buildSegments: {
+ '1m': RowStatsCounter;
+ '5m': RowStatsCounter;
+ '15m': RowStatsCounter;
+ };
+ };
+ totals: {
+ buildSegments: RowStatsCounter;
+ };
+}
+
+export interface RowStatsCounter {
+ processed: number;
+ processedBytes: number;
+ processedWithError: number;
+ thrownAway: number;
+ unparseable: number;
+}
+
+function sumRowStatsCounter(rowStats: RowStatsCounter[]): RowStatsCounter {
+ return {
+ processed: sum(rowStats, d => d.processed),
+ processedBytes: sum(rowStats, d => d.processedBytes),
+ processedWithError: sum(rowStats, d => d.processedWithError),
+ thrownAway: sum(rowStats, d => d.thrownAway),
+ unparseable: sum(rowStats, d => d.unparseable),
+ };
+}
+
+function maxRowStatsCounter(rowStats: RowStatsCounter[]): RowStatsCounter {
+ return {
+ processed: max(rowStats, d => d.processed) ?? 0,
+ processedBytes: max(rowStats, d => d.processedBytes) ?? 0,
+ processedWithError: max(rowStats, d => d.processedWithError) ?? 0,
+ thrownAway: max(rowStats, d => d.thrownAway) ?? 0,
+ unparseable: max(rowStats, d => d.unparseable) ?? 0,
+ };
+}
+
+function getRowStatsCounter(rowStats: RowStats, key: RowStatsKey):
RowStatsCounter | undefined {
+ if (key === 'totals') {
+ return deepGet(rowStats, 'totals.buildSegments');
+ } else {
+ return deepGet(rowStats, `movingAverages.buildSegments.${key}`);
+ }
+}
+
+export function getTotalSupervisorStats(
+ stats: SupervisorStats,
+ key: RowStatsKey,
+ activeTaskIds: string[] | undefined,
+): RowStatsCounter {
+ return sumRowStatsCounter(
+ Object.values(stats).map(s =>
+ maxRowStatsCounter(
+ filterMap(Object.entries(s), ([taskId, rs]) =>
+ !activeTaskIds || activeTaskIds.includes(taskId)
+ ? getRowStatsCounter(rs, key)
+ : undefined,
+ ),
+ ),
+ ),
+ );
+}
diff --git a/web-console/src/utils/druid-query.ts
b/web-console/src/utils/druid-query.ts
index c94bfca3d1c..15410329704 100644
--- a/web-console/src/utils/druid-query.ts
+++ b/web-console/src/utils/druid-query.ts
@@ -17,7 +17,7 @@
*/
import { C } from '@druid-toolkit/query';
-import type { AxiosResponse } from 'axios';
+import type { AxiosResponse, CancelToken } from 'axios';
import axios from 'axios';
import { Api } from '../singletons';
@@ -329,10 +329,13 @@ export async function queryDruidRune(runeQuery:
Record<string, any>): Promise<an
return runeResultResp.data;
}
-export async function queryDruidSql<T = any>(sqlQueryPayload: Record<string,
any>): Promise<T[]> {
+export async function queryDruidSql<T = any>(
+ sqlQueryPayload: Record<string, any>,
+ cancelToken?: CancelToken,
+): Promise<T[]> {
let sqlResultResp: AxiosResponse;
try {
- sqlResultResp = await Api.instance.post('/druid/v2/sql', sqlQueryPayload);
+ sqlResultResp = await Api.instance.post('/druid/v2/sql', sqlQueryPayload,
{ cancelToken });
} catch (e) {
throw new Error(getDruidErrorMessage(e));
}
diff --git a/web-console/src/utils/general.tsx
b/web-console/src/utils/general.tsx
index 3a770c67630..b4537a63e08 100644
--- a/web-console/src/utils/general.tsx
+++ b/web-console/src/utils/general.tsx
@@ -239,14 +239,26 @@ export function formatNumber(n: NumberLike): string {
return n.toLocaleString('en-US', { maximumFractionDigits: 20 });
}
+export function formatRate(n: NumberLike) {
+ return numeral(n).format('0,0.0') + '/s';
+}
+
export function formatBytes(n: NumberLike): string {
return numeral(n).format('0.00 b');
}
+export function formatByteRate(n: NumberLike): string {
+ return numeral(n).format('0.00 b') + '/s';
+}
+
export function formatBytesCompact(n: NumberLike): string {
return numeral(n).format('0.00b');
}
+export function formatByteRateCompact(n: NumberLike): string {
+ return numeral(n).format('0.00b') + '/s';
+}
+
export function formatMegabytes(n: NumberLike): string {
return numeral(Number(n) / 1048576).format('0,0.0');
}
diff --git a/web-console/src/utils/local-storage-backed-visibility.tsx
b/web-console/src/utils/local-storage-backed-visibility.tsx
index c335180056b..f20031f2b8d 100644
--- a/web-console/src/utils/local-storage-backed-visibility.tsx
+++ b/web-console/src/utils/local-storage-backed-visibility.tsx
@@ -65,7 +65,7 @@ export class LocalStorageBackedVisibility {
return new LocalStorageBackedVisibility(this.key, defaultHidden,
newVisibility);
}
- public shown(value: string): boolean {
- return this.visibility[value] ?? !this.defaultHidden.includes(value);
+ public shown(...values: string[]): boolean {
+ return values.some(value => this.visibility[value] ??
!this.defaultHidden.includes(value));
}
}
diff --git a/web-console/src/utils/table-helpers.ts
b/web-console/src/utils/table-helpers.ts
index e864aef131f..7eedd1acaab 100644
--- a/web-console/src/utils/table-helpers.ts
+++ b/web-console/src/utils/table-helpers.ts
@@ -17,6 +17,8 @@
*/
import type { QueryResult } from '@druid-toolkit/query';
+import { C } from '@druid-toolkit/query';
+import type { Filter } from 'react-table';
import { filterMap, formatNumber, oneOf } from './general';
import { deepSet } from './object-change';
@@ -56,3 +58,20 @@ export function getNumericColumnBraces(
return numericColumnBraces;
}
+
+export interface Sorted {
+ id: string;
+ desc: boolean;
+}
+
+export interface TableState {
+ page: number;
+ pageSize: number;
+ filtered: Filter[];
+ sorted: Sorted[];
+}
+
+export function sortedToOrderByClause(sorted: Sorted[]): string | undefined {
+ if (!sorted.length) return;
+ return 'ORDER BY ' + sorted.map(sort => `${C(sort.id)} ${sort.desc ? 'DESC'
: 'ASC'}`).join(', ');
+}
diff --git
a/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
b/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
index 2cd926e01f9..b627b2e500c 100644
---
a/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
+++
b/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
@@ -81,7 +81,6 @@ exports[`DatasourcesView matches snapshot 1`] = `
"% Compacted",
"Left to be compacted",
"Retention",
- "Actions",
]
}
onChange={[Function]}
@@ -338,7 +337,7 @@ exports[`DatasourcesView matches snapshot 1`] = `
"accessor": "datasource",
"filterable": false,
"id": "actions",
- "show": true,
+ "sortable": false,
"width": 70,
},
]
diff --git a/web-console/src/views/datasources-view/datasources-view.tsx
b/web-console/src/views/datasources-view/datasources-view.tsx
index 75541b82999..713df9b18b1 100644
--- a/web-console/src/views/datasources-view/datasources-view.tsx
+++ b/web-console/src/views/datasources-view/datasources-view.tsx
@@ -102,7 +102,6 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
'% Compacted',
'Left to be compacted',
'Retention',
- ACTION_COLUMN_LABEL,
],
'no-sql': [
'Datasource name',
@@ -114,7 +113,6 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
'% Compacted',
'Left to be compacted',
'Retention',
- ACTION_COLUMN_LABEL,
],
'no-proxy': [
'Datasource name',
@@ -128,7 +126,6 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
'Total rows',
'Avg. row size',
'Replicated size',
- ACTION_COLUMN_LABEL,
],
};
@@ -338,12 +335,11 @@ export class DatasourcesView extends React.PureComponent<
const columns = compact(
[
visibleColumns.shown('Datasource name') && `datasource`,
- (visibleColumns.shown('Availability') || visibleColumns.shown('Segment
granularity')) && [
+ visibleColumns.shown('Availability', 'Segment granularity') && [
`COUNT(*) FILTER (WHERE is_active = 1) AS num_segments`,
`COUNT(*) FILTER (WHERE is_published = 1 AND is_overshadowed = 0 AND
replication_factor = 0) AS num_zero_replica_segments`,
],
- (visibleColumns.shown('Availability') ||
- visibleColumns.shown('Historical load/drop queues')) && [
+ visibleColumns.shown('Availability', 'Historical load/drop queues') &&
[
`COUNT(*) FILTER (WHERE is_published = 1 AND is_overshadowed = 0 AND
is_available = 0 AND replication_factor > 0) AS num_segments_to_load`,
`COUNT(*) FILTER (WHERE is_available = 1 AND is_active = 0) AS
num_segments_to_drop`,
],
@@ -1577,11 +1573,11 @@ GROUP BY 1, 2`;
},
{
Header: ACTION_COLUMN_LABEL,
- show: visibleColumns.shown(ACTION_COLUMN_LABEL),
accessor: 'datasource',
id: ACTION_COLUMN_ID,
width: ACTION_COLUMN_WIDTH,
filterable: false,
+ sortable: false,
Cell: ({ value: datasource, original }) => {
const { unused, rules, compaction } = original as Datasource;
const datasourceActions = this.getDatasourceActions(
diff --git a/web-console/src/views/load-data-view/info-messages.tsx
b/web-console/src/views/load-data-view/info-messages.tsx
index b88cf8a70c2..ad9e96667db 100644
--- a/web-console/src/views/load-data-view/info-messages.tsx
+++ b/web-console/src/views/load-data-view/info-messages.tsx
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import { Button, Callout, Code, FormGroup, Intent } from '@blueprintjs/core';
+import { Button, Callout, Code, FormGroup, Intent, Tag } from
'@blueprintjs/core';
import React from 'react';
import { ExternalLink, LearnMore } from '../../components';
@@ -236,8 +236,8 @@ export const AppendToExistingIssue = React.memo(function
AppendToExistingIssue(
<FormGroup>
<Callout intent={Intent.DANGER}>
<p>
- Only <Code>dynamic</Code> partitioning supports
<Code>appendToExisting: true</Code>. You
- have currently selected <Code>{partitionsSpecType}</Code>
partitioning.
+ Only <Tag minimal>dynamic</Tag> partitioning supports
<Code>appendToExisting: true</Code>.
+ You have currently selected <Tag minimal>{partitionsSpecType}</Tag>
partitioning.
</p>
<Button
intent={Intent.SUCCESS}
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx
b/web-console/src/views/load-data-view/load-data-view.tsx
index 2bd3623fa0e..5e907a24267 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -34,6 +34,7 @@ import {
Radio,
RadioGroup,
Switch,
+ Tag,
TextArea,
} from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
@@ -3073,8 +3074,9 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
<p>Your partitioning and sorting configuration is uncommon.</p>
<p>
For best performance the first dimension in your schema (
- <Code>{firstDimensionName}</Code>), which is what the data
will be primarily sorted
- on, commonly matches the partitioning dimension
(<Code>{partitionDimension}</Code>).
+ <Tag minimal>{firstDimensionName}</Tag>), which is what the
data will be primarily
+ sorted on, commonly matches the partitioning dimension (
+ <Tag minimal>{partitionDimension}</Tag>).
</p>
<p>
<Button
@@ -3451,11 +3453,11 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
<p>
You have enabled type-aware schema discovery (
<Code>useSchemaDiscovery: true</Code>) to ingest data into
the existing
- datasource <Code>{datasource}</Code>.
+ datasource <Tag minimal>{datasource}</Tag>.
</p>
<p>
If you used string-based schema discovery when first
ingesting data to{' '}
- <Code>{datasource}</Code>, using type-aware schema
discovery now can cause
+ <Tag minimal>{datasource}</Tag>, using type-aware schema
discovery now can cause
problems with the values multi-value string dimensions.
</p>
<p>
diff --git
a/web-console/src/views/lookups-view/__snapshots__/lookups-view.spec.tsx.snap
b/web-console/src/views/lookups-view/__snapshots__/lookups-view.spec.tsx.snap
index f612725e298..07fbc786424 100755
---
a/web-console/src/views/lookups-view/__snapshots__/lookups-view.spec.tsx.snap
+++
b/web-console/src/views/lookups-view/__snapshots__/lookups-view.spec.tsx.snap
@@ -25,7 +25,6 @@ exports[`LookupsView matches snapshot 1`] = `
"Version",
"Poll period",
"Summary",
- "Actions",
]
}
onChange={[Function]}
@@ -148,7 +147,7 @@ exports[`LookupsView matches snapshot 1`] = `
"accessor": "id",
"filterable": false,
"id": "actions",
- "show": true,
+ "sortable": false,
"width": 70,
},
]
diff --git a/web-console/src/views/lookups-view/lookups-view.tsx
b/web-console/src/views/lookups-view/lookups-view.tsx
index 8f19e55d4ca..af8207f6ab1 100644
--- a/web-console/src/views/lookups-view/lookups-view.tsx
+++ b/web-console/src/views/lookups-view/lookups-view.tsx
@@ -60,7 +60,6 @@ const tableColumns: string[] = [
'Version',
'Poll period',
'Summary',
- ACTION_COLUMN_LABEL,
];
const DEFAULT_LOOKUP_TIER = '__default';
@@ -442,10 +441,10 @@ export class LookupsView extends
React.PureComponent<LookupsViewProps, LookupsVi
},
{
Header: ACTION_COLUMN_LABEL,
- show: visibleColumns.shown(ACTION_COLUMN_LABEL),
id: ACTION_COLUMN_ID,
width: ACTION_COLUMN_WIDTH,
filterable: false,
+ sortable: false,
accessor: 'id',
Cell: ({ original }) => {
const lookupId = original.id;
diff --git
a/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap
b/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap
index d37f1ebe3e6..9c7e40197a6 100755
---
a/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap
+++
b/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap
@@ -70,7 +70,6 @@ exports[`SegmentsView matches snapshot 1`] = `
"Is realtime",
"Is published",
"Is overshadowed",
- "Actions",
]
}
onChange={[Function]}
@@ -356,6 +355,7 @@ exports[`SegmentsView matches snapshot 1`] = `
"filterable": false,
"id": "actions",
"show": true,
+ "sortable": false,
"width": 70,
},
]
diff --git a/web-console/src/views/segments-view/segments-view.tsx
b/web-console/src/views/segments-view/segments-view.tsx
index ae40a8d641e..7d7bcaeec46 100644
--- a/web-console/src/views/segments-view/segments-view.tsx
+++ b/web-console/src/views/segments-view/segments-view.tsx
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import { Button, ButtonGroup, Code, Intent, Label, MenuItem, Switch } from
'@blueprintjs/core';
+import { Button, ButtonGroup, Intent, Label, MenuItem, Switch, Tag } from
'@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import { C, L, SqlComparison, SqlExpression } from '@druid-toolkit/query';
import classNames from 'classnames';
@@ -53,7 +53,7 @@ import {
STANDARD_TABLE_PAGE_SIZE_OPTIONS,
} from '../../react-table';
import { Api } from '../../singletons';
-import type { NumberLike } from '../../utils';
+import type { NumberLike, TableState } from '../../utils';
import {
compact,
countBy,
@@ -69,6 +69,7 @@ import {
queryDruidSql,
QueryManager,
QueryState,
+ sortedToOrderByClause,
twoLines,
} from '../../utils';
import type { BasicAction } from '../../utils/basic-action';
@@ -96,18 +97,8 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
'Is realtime',
'Is published',
'Is overshadowed',
- ACTION_COLUMN_LABEL,
- ],
- 'no-sql': [
- 'Segment ID',
- 'Datasource',
- 'Start',
- 'End',
- 'Version',
- 'Partition',
- 'Size',
- ACTION_COLUMN_LABEL,
],
+ 'no-sql': ['Segment ID', 'Datasource', 'Start', 'End', 'Version',
'Partition', 'Size'],
'no-proxy': [
'Segment ID',
'Datasource',
@@ -134,23 +125,6 @@ function formatRangeDimensionValue(dimension: any, value:
any): string {
return `${C(String(dimension))}=${L(String(value))}`;
}
-interface Sorted {
- id: string;
- desc: boolean;
-}
-
-function sortedToOrderByClause(sorted: Sorted[]): string | undefined {
- if (!sorted.length) return;
- return 'ORDER BY ' + sorted.map(sort => `${C(sort.id)} ${sort.desc ? 'DESC'
: 'ASC'}`).join(', ');
-}
-
-interface TableState {
- page: number;
- pageSize: number;
- filtered: Filter[];
- sorted: Sorted[];
-}
-
interface SegmentsQuery extends TableState {
visibleColumns: LocalStorageBackedVisibility;
capabilities: Capabilities;
@@ -217,7 +191,7 @@ export class SegmentsView extends
React.PureComponent<SegmentsViewProps, Segment
WHEN "start" LIKE '%:00.000Z' AND "end" LIKE '%:00.000Z' THEN 'Minute'
ELSE 'Sub minute'
END AS "time_span"`,
- (visibleColumns.shown('Shard type') || visibleColumns.shown('Shard
spec')) && `"shard_spec"`,
+ visibleColumns.shown('Shard type', 'Shard spec') && `"shard_spec"`,
visibleColumns.shown('Partition') && `"partition_num"`,
visibleColumns.shown('Size') && `"size"`,
visibleColumns.shown('Num rows') && `"num_rows"`,
@@ -471,7 +445,8 @@ END AS "time_span"`,
const { capabilities } = this.props;
const { visibleColumns } = this.state;
if (tableState) this.lastTableState = tableState;
- const { page, pageSize, filtered, sorted } = this.lastTableState!;
+ if (!this.lastTableState) return;
+ const { page, pageSize, filtered, sorted } = this.lastTableState;
this.segmentsQueryManager.runQuery({
page,
pageSize,
@@ -895,11 +870,12 @@ END AS "time_span"`,
},
{
Header: ACTION_COLUMN_LABEL,
- show: capabilities.hasCoordinatorAccess() &&
visibleColumns.shown(ACTION_COLUMN_LABEL),
+ show: capabilities.hasCoordinatorAccess(),
id: ACTION_COLUMN_ID,
accessor: 'segment_id',
width: ACTION_COLUMN_WIDTH,
filterable: false,
+ sortable: false,
Cell: row => {
if (row.aggregated) return '';
const id = row.value;
@@ -935,7 +911,7 @@ END AS "time_span"`,
);
return resp.data;
}}
- confirmButtonText="Drop Segment"
+ confirmButtonText="Drop segment"
successText="Segment drop request acknowledged, next time the
coordinator runs segment will be dropped"
failText="Could not drop segment"
intent={Intent.DANGER}
@@ -947,7 +923,7 @@ END AS "time_span"`,
}}
>
<p>
- Are you sure you want to drop segment
<Code>{terminateSegmentId}</Code>?
+ Are you sure you want to drop segment <Tag
minimal>{terminateSegmentId}</Tag>?
</p>
<p>This action is not reversible.</p>
</AsyncActionDialog>
diff --git
a/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap
b/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap
index baedf5165d6..93e47b06e27 100644
---
a/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap
+++
b/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap
@@ -60,7 +60,6 @@ exports[`ServicesView renders data 1`] = `
"Usage",
"Start time",
"Detail",
- "Actions",
]
}
onChange={[Function]}
@@ -224,6 +223,7 @@ exports[`ServicesView renders data 1`] = `
"filterable": false,
"id": "actions",
"show": true,
+ "sortable": false,
"width": 70,
},
]
diff --git a/web-console/src/views/services-view/services-view.tsx
b/web-console/src/views/services-view/services-view.tsx
index fd9c4caef24..3ff6eead276 100644
--- a/web-console/src/views/services-view/services-view.tsx
+++ b/web-console/src/views/services-view/services-view.tsx
@@ -71,7 +71,6 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
'Usage',
'Start time',
'Detail',
- ACTION_COLUMN_LABEL,
],
'no-sql': [
'Service',
@@ -83,7 +82,6 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
'Max size',
'Usage',
'Detail',
- ACTION_COLUMN_LABEL,
],
'no-proxy': [
'Service',
@@ -646,11 +644,12 @@ ORDER BY
},
{
Header: ACTION_COLUMN_LABEL,
- show: capabilities.hasOverlordAccess() &&
visibleColumns.shown(ACTION_COLUMN_LABEL),
+ show: capabilities.hasOverlordAccess(),
id: ACTION_COLUMN_ID,
width: ACTION_COLUMN_WIDTH,
accessor: row => row.workerInfo,
filterable: false,
+ sortable: false,
Cell: ({ value, aggregated }) => {
if (aggregated) return '';
if (!value) return null;
diff --git
a/web-console/src/views/supervisors-view/__snapshots__/supervisors-view.spec.tsx.snap
b/web-console/src/views/supervisors-view/__snapshots__/supervisors-view.spec.tsx.snap
index 6053d75102e..f43b9d3eb90 100644
---
a/web-console/src/views/supervisors-view/__snapshots__/supervisors-view.spec.tsx.snap
+++
b/web-console/src/views/supervisors-view/__snapshots__/supervisors-view.spec.tsx.snap
@@ -78,8 +78,23 @@ exports[`SupervisorsView matches snapshot 1`] = `
"Type",
"Topic/Stream",
"Status",
- "Running tasks",
- "Actions",
+ "Configured tasks",
+ {
+ "label": "status API",
+ "text": "Running tasks",
+ },
+ {
+ "label": "status API",
+ "text": "Aggregate lag",
+ },
+ {
+ "label": "status API",
+ "text": "Recent errors",
+ },
+ {
+ "label": "stats API",
+ "text": "Stats",
+ },
]
}
onChange={[Function]}
@@ -155,37 +170,150 @@ exports[`SupervisorsView matches snapshot 1`] = `
"accessor": "supervisor_id",
"id": "supervisor_id",
"show": true,
- "width": 300,
+ "width": 280,
},
{
"Cell": [Function],
"Header": "Type",
"accessor": "type",
"show": true,
- "width": 100,
+ "width": 80,
},
{
"Cell": [Function],
"Header": "Topic/Stream",
"accessor": "source",
"show": true,
- "width": 300,
+ "width": 200,
},
{
"Cell": [Function],
"Header": "Status",
"accessor": "detailed_state",
- "id": "status",
+ "id": "detailed_state",
+ "show": true,
+ "width": 130,
+ },
+ {
+ "Cell": [Function],
+ "Header": "Configured tasks",
+ "accessor": "spec",
+ "className": "padded",
+ "filterable": false,
+ "id": "configured_tasks",
"show": true,
- "width": 250,
+ "sortable": false,
+ "width": 130,
},
{
"Cell": [Function],
"Header": "Running tasks",
- "accessor": "running_tasks",
+ "accessor": "status.payload",
"filterable": false,
"id": "running_tasks",
"show": true,
+ "sortable": false,
+ "width": 150,
+ },
+ {
+ "Cell": [Function],
+ "Header": "Aggregate lag",
+ "accessor": "status.payload.aggregateLag",
+ "className": "padded",
+ "filterable": false,
+ "show": true,
+ "sortable": false,
+ "width": 200,
+ },
+ {
+ "Cell": [Function],
+ "Header": <React.Fragment>
+ Stats
+ <br />
+ <Blueprint4.Popover2
+ boundary="clippingParents"
+ captureDismiss={false}
+ content={
+ <Blueprint4.Menu>
+ <Blueprint4.MenuItem
+ active={false}
+ disabled={false}
+ icon="circle"
+ multiline={false}
+ onClick={[Function]}
+ popoverProps={{}}
+ selected={false}
+ shouldDismissPopover={true}
+ text="Rate over past 1 minute"
+ />
+ <Blueprint4.MenuItem
+ active={false}
+ disabled={false}
+ icon="tick-circle"
+ multiline={false}
+ onClick={[Function]}
+ popoverProps={{}}
+ selected={false}
+ shouldDismissPopover={true}
+ text="Rate over past 5 minutes"
+ />
+ <Blueprint4.MenuItem
+ active={false}
+ disabled={false}
+ icon="circle"
+ multiline={false}
+ onClick={[Function]}
+ popoverProps={{}}
+ selected={false}
+ shouldDismissPopover={true}
+ text="Rate over past 15 minutes"
+ />
+ </Blueprint4.Menu>
+ }
+ defaultIsOpen={false}
+ disabled={false}
+ fill={false}
+ hasBackdrop={false}
+ hoverCloseDelay={300}
+ hoverOpenDelay={150}
+ inheritDarkTheme={true}
+ interactionKind="click"
+ matchTargetWidth={false}
+ minimal={false}
+ openOnTargetFocus={true}
+ position="bottom"
+ positioningStrategy="absolute"
+ shouldReturnFocusOnClose={false}
+ targetTagName="span"
+ transitionDuration={300}
+ usePortal={true}
+ >
+ <i
+ className="title-button"
+ >
+ Rate over past 5 minutes
+
+ <Blueprint4.Icon
+ icon="caret-down"
+ />
+ </i>
+ </Blueprint4.Popover2>
+ </React.Fragment>,
+ "accessor": "stats",
+ "className": "padded",
+ "filterable": false,
+ "id": "stats",
+ "show": true,
+ "sortable": false,
+ "width": 300,
+ },
+ {
+ "Cell": [Function],
+ "Header": "Recent errors",
+ "accessor": "status.payload.recentErrors",
+ "filterable": false,
+ "show": true,
+ "sortable": false,
"width": 150,
},
{
@@ -194,7 +322,7 @@ exports[`SupervisorsView matches snapshot 1`] = `
"accessor": "supervisor_id",
"filterable": false,
"id": "actions",
- "show": true,
+ "sortable": false,
"width": 70,
},
]
@@ -244,7 +372,7 @@ exports[`SupervisorsView matches snapshot 1`] = `
getTrProps={[Function]}
groupedByPivotKey="_groupedByPivot"
indexKey="_index"
- loading={true}
+ loading={false}
loadingText="Loading..."
multiSort={true}
nestingLevelKey="_nestingLevel"
diff --git a/web-console/src/views/supervisors-view/supervisors-view.scss
b/web-console/src/views/supervisors-view/supervisors-view.scss
index edf04bc4e0d..25c55e69b44 100644
--- a/web-console/src/views/supervisors-view/supervisors-view.scss
+++ b/web-console/src/views/supervisors-view/supervisors-view.scss
@@ -28,5 +28,18 @@
top: $view-control-bar-height + $standard-padding;
bottom: 0;
width: 100%;
+
+ .title-button {
+ cursor: pointer;
+ }
+
+ .detail-line {
+ font-style: italic;
+ opacity: 0.6;
+ }
+
+ .warning-line {
+ color: $orange4;
+ }
}
}
diff --git a/web-console/src/views/supervisors-view/supervisors-view.tsx
b/web-console/src/views/supervisors-view/supervisors-view.tsx
index fef74fbeebe..f6604445587 100644
--- a/web-console/src/views/supervisors-view/supervisors-view.tsx
+++ b/web-console/src/views/supervisors-view/supervisors-view.tsx
@@ -16,12 +16,16 @@
* limitations under the License.
*/
-import { Code, Intent, MenuItem } from '@blueprintjs/core';
+import { Icon, Intent, Menu, MenuItem, Position, Tag } from
'@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
+import { Popover2 } from '@blueprintjs/popover2';
+import * as JSONBig from 'json-bigint-native';
+import type { JSX } from 'react';
import React from 'react';
import type { Filter } from 'react-table';
import ReactTable from 'react-table';
+import type { TableColumnSelectorColumn } from '../../components';
import {
ACTION_COLUMN_ID,
ACTION_COLUMN_LABEL,
@@ -41,39 +45,72 @@ import {
SupervisorTableActionDialog,
} from '../../dialogs';
import { SupervisorResetOffsetsDialog } from
'../../dialogs/supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog';
-import type { QueryWithContext } from '../../druid-models';
+import type {
+ IngestionSpec,
+ QueryWithContext,
+ RowStatsKey,
+ SupervisorStatus,
+ SupervisorStatusTask,
+} from '../../druid-models';
+import { getTotalSupervisorStats } from '../../druid-models';
import type { Capabilities } from '../../helpers';
-import { SMALL_TABLE_PAGE_SIZE, SMALL_TABLE_PAGE_SIZE_OPTIONS } from
'../../react-table';
+import {
+ SMALL_TABLE_PAGE_SIZE,
+ SMALL_TABLE_PAGE_SIZE_OPTIONS,
+ sqlQueryCustomTableFilter,
+} from '../../react-table';
import { Api, AppToaster } from '../../singletons';
+import type { TableState } from '../../utils';
import {
+ assemble,
+ checkedCircleIcon,
deepGet,
+ formatByteRate,
+ formatBytes,
+ formatInteger,
+ formatRate,
getDruidErrorMessage,
- groupByAsMap,
hasPopoverOpen,
LocalStorageBackedVisibility,
LocalStorageKeys,
- lookupBy,
+ nonEmptyArray,
oneOf,
pluralIfNeeded,
queryDruidSql,
QueryManager,
QueryState,
+ sortedToOrderByClause,
twoLines,
} from '../../utils';
import type { BasicAction } from '../../utils/basic-action';
import './supervisors-view.scss';
-const supervisorTableColumns: string[] = [
+const SUPERVISOR_TABLE_COLUMNS: TableColumnSelectorColumn[] = [
'Supervisor ID',
'Type',
'Topic/Stream',
'Status',
- 'Running tasks',
- ACTION_COLUMN_LABEL,
+ 'Configured tasks',
+ { text: 'Running tasks', label: 'status API' },
+ { text: 'Aggregate lag', label: 'status API' },
+ { text: 'Recent errors', label: 'status API' },
+ { text: 'Stats', label: 'stats API' },
];
-interface SupervisorQuery {
+const ROW_STATS_KEYS: RowStatsKey[] = ['1m', '5m', '15m'];
+const STATUS_API_TIMEOUT = 5000;
+const STATS_API_TIMEOUT = 5000;
+
+function getRowStatsKeyTitle(key: RowStatsKey) {
+ return `Rate over past ${pluralIfNeeded(parseInt(key, 10), 'minute')}`;
+}
+
+function getRowStatsKeySeconds(key: RowStatsKey): number {
+ return parseInt(key, 10) * 60;
+}
+
+interface SupervisorQuery extends TableState {
capabilities: Capabilities;
visibleColumns: LocalStorageBackedVisibility;
}
@@ -83,14 +120,10 @@ interface SupervisorQueryResultRow {
type: string;
source: string;
detailed_state: string;
+ spec?: IngestionSpec;
suspended: boolean;
- running_tasks?: number;
-}
-
-interface RunningTaskRow {
- datasource: string;
- type: string;
- num_running_tasks: number;
+ status?: SupervisorStatus;
+ stats?: any;
}
export interface SupervisorsViewProps {
@@ -106,6 +139,7 @@ export interface SupervisorsViewProps {
export interface SupervisorsViewState {
supervisorsState: QueryState<SupervisorQueryResultRow[]>;
+ statsKey: RowStatsKey;
resumeSupervisorId?: string;
suspendSupervisorId?: string;
@@ -165,25 +199,12 @@ export class SupervisorsView extends React.PureComponent<
SupervisorQueryResultRow[]
>;
- static SUPERVISOR_SQL = `SELECT
- "supervisor_id",
- "type",
- "source",
- CASE WHEN "suspended" = 0 THEN "detailed_state" ELSE 'SUSPENDED' END AS
"detailed_state",
- "suspended" = 1 AS "suspended"
-FROM "sys"."supervisors"
-ORDER BY "supervisor_id"`;
-
- static RUNNING_TASK_SQL = `SELECT
- "datasource", "type", COUNT(*) AS "num_running_tasks"
-FROM "sys"."tasks" WHERE "status" = 'RUNNING' AND "runner_status" = 'RUNNING'
-GROUP BY 1, 2`;
-
constructor(props: SupervisorsViewProps) {
super(props);
this.state = {
supervisorsState: QueryState.INIT,
+ statsKey: '5m',
showResumeAllSupervisors: false,
showSuspendAllSupervisors: false,
@@ -199,14 +220,49 @@ GROUP BY 1, 2`;
};
this.supervisorQueryManager = new QueryManager({
- processQuery: async ({ capabilities, visibleColumns }) => {
+ processQuery: async (
+ { capabilities, visibleColumns, filtered, sorted, page, pageSize },
+ cancelToken,
+ setIntermediateQuery,
+ ) => {
let supervisors: SupervisorQueryResultRow[];
if (capabilities.hasSql()) {
- supervisors = await queryDruidSql<SupervisorQueryResultRow>({
- query: SupervisorsView.SUPERVISOR_SQL,
- });
+ const sqlQuery = assemble(
+ 'WITH s AS (SELECT',
+ ' "supervisor_id",',
+ ' "type",',
+ ' "source",',
+ ` CASE WHEN "suspended" = 0 THEN "detailed_state" ELSE
'SUSPENDED' END AS "detailed_state",`,
+ visibleColumns.shown('Configured tasks') ? ' "spec",' : undefined,
+ ' "suspended" = 1 AS "suspended"',
+ 'FROM "sys"."supervisors")',
+ 'SELECT *',
+ 'FROM s',
+ filtered.length
+ ? `WHERE ${filtered.map(sqlQueryCustomTableFilter).join(' AND
')}`
+ : undefined,
+ sortedToOrderByClause(sorted),
+ `LIMIT ${pageSize}`,
+ page ? `OFFSET ${page * pageSize}` : undefined,
+ ).join('\n');
+ setIntermediateQuery(sqlQuery);
+ supervisors = await queryDruidSql<SupervisorQueryResultRow>(
+ {
+ query: sqlQuery,
+ },
+ cancelToken,
+ );
+
+ for (const supervisor of supervisors) {
+ const spec: any = supervisor.spec;
+ if (typeof spec === 'string') {
+ supervisor.spec = JSONBig.parse(spec);
+ }
+ }
} else if (capabilities.hasOverlordAccess()) {
- const supervisorList = (await
Api.instance.get('/druid/indexer/v1/supervisor?full')).data;
+ const supervisorList = (
+ await Api.instance.get('/druid/indexer/v1/supervisor?full', {
cancelToken })
+ ).data;
if (!Array.isArray(supervisorList)) {
throw new Error(`Unexpected result from
/druid/indexer/v1/supervisor?full`);
}
@@ -220,48 +276,69 @@ GROUP BY 1, 2`;
'n/a',
state: deepGet(sup, 'state'),
detailed_state: deepGet(sup, 'detailedState'),
+ spec: sup.spec,
suspended: Boolean(deepGet(sup, 'suspended')),
};
});
+
+ const firstSorted = sorted[0];
+ if (firstSorted) {
+ const { id, desc } = firstSorted;
+ supervisors.sort((s1: any, s2: any) => {
+ return (
+ String(s1[id]).localeCompare(String(s2[id]), undefined, {
numeric: true }) *
+ (desc ? -1 : 1)
+ );
+ });
+ }
} else {
throw new Error(`must have SQL or overlord access`);
}
- if (visibleColumns.shown('Running tasks')) {
- try {
- let runningTaskLookup: Record<string, number>;
- if (capabilities.hasSql()) {
- const runningTasks = await queryDruidSql<RunningTaskRow>({
- query: SupervisorsView.RUNNING_TASK_SQL,
- });
-
- runningTaskLookup = lookupBy(
- runningTasks,
- ({ datasource, type }) => `${datasource}_${type}`,
- ({ num_running_tasks }) => num_running_tasks,
- );
- } else if (capabilities.hasOverlordAccess()) {
- const taskList = (await
Api.instance.get(`/druid/indexer/v1/tasks?state=running`))
- .data;
- runningTaskLookup = groupByAsMap(
- taskList,
- (t: any) => `${t.dataSource}_${t.type}`,
- xs => xs.length,
- );
- } else {
- throw new Error(`must have SQL or overlord access`);
- }
-
- supervisors.forEach(supervisor => {
- supervisor.running_tasks =
-
runningTaskLookup[`${supervisor.supervisor_id}_index_${supervisor.type}`] || 0;
- });
- } catch (e) {
+ if (capabilities.hasOverlordAccess()) {
+ let showIssue = (message: string) => {
+ showIssue = () => {}; // Only show once
AppToaster.show({
icon: IconNames.ERROR,
intent: Intent.DANGER,
- message: 'Could not get running task counts',
+ message,
});
+ };
+
+ if (visibleColumns.shown('Running tasks', 'Aggregate lag', 'Recent
errors')) {
+ try {
+ for (const supervisor of supervisors) {
+ cancelToken.throwIfRequested();
+ supervisor.status = (
+ await Api.instance.get(
+ `/druid/indexer/v1/supervisor/${Api.encodePath(
+ supervisor.supervisor_id,
+ )}/status`,
+ { cancelToken, timeout: STATUS_API_TIMEOUT },
+ )
+ ).data;
+ }
+ } catch (e) {
+ showIssue('Could not get status');
+ }
+ }
+
+ if (visibleColumns.shown('Stats')) {
+ try {
+ for (const supervisor of supervisors) {
+ cancelToken.throwIfRequested();
+ supervisor.stats = (
+ await Api.instance.get(
+ `/druid/indexer/v1/supervisor/${Api.encodePath(
+ supervisor.supervisor_id,
+ )}/stats`,
+ { cancelToken, timeout: STATS_API_TIMEOUT },
+ )
+ ).data;
+ }
+ } catch (e) {
+ showIssue('Could not get stats');
+ }
}
}
@@ -275,17 +352,28 @@ GROUP BY 1, 2`;
});
}
- componentDidMount(): void {
- const { capabilities } = this.props;
- const { visibleColumns } = this.state;
-
- this.supervisorQueryManager.runQuery({ capabilities, visibleColumns:
visibleColumns });
- }
+ private lastTableState: TableState | undefined;
componentWillUnmount(): void {
this.supervisorQueryManager.terminate();
}
+ private readonly fetchData = (tableState?: TableState) => {
+ const { capabilities } = this.props;
+ const { visibleColumns } = this.state;
+ if (tableState) this.lastTableState = tableState;
+ if (!this.lastTableState) return;
+ const { page, pageSize, filtered, sorted } = this.lastTableState;
+ this.supervisorQueryManager.runQuery({
+ page,
+ pageSize,
+ filtered,
+ sorted,
+ visibleColumns,
+ capabilities,
+ });
+ };
+
private readonly closeSpecDialogs = () => {
this.setState({
supervisorSpecDialogOpen: false,
@@ -389,7 +477,7 @@ GROUP BY 1, 2`;
}}
>
<p>
- Are you sure you want to resume supervisor
<Code>{resumeSupervisorId}</Code>?
+ Are you sure you want to resume supervisor <Tag
minimal>{resumeSupervisorId}</Tag>?
</p>
</AsyncActionDialog>
);
@@ -420,7 +508,7 @@ GROUP BY 1, 2`;
}}
>
<p>
- Are you sure you want to suspend supervisor
<Code>{suspendSupervisorId}</Code>?
+ Are you sure you want to suspend supervisor <Tag
minimal>{suspendSupervisorId}</Tag>?
</p>
</AsyncActionDialog>
);
@@ -465,17 +553,20 @@ GROUP BY 1, 2`;
this.supervisorQueryManager.rerunLastQuery();
}}
warningChecks={[
- `I understand that resetting ${resetSupervisorId} will clear
checkpoints and therefore lead to data loss or duplication.`,
+ <>
+ I understand that resetting <Tag minimal>{resetSupervisorId}</Tag>
will clear
+ checkpoints and may lead to data loss or duplication.
+ </>,
'I understand that this operation cannot be undone.',
]}
>
<p>
- Are you sure you want to hard reset supervisor
<Code>{resetSupervisorId}</Code>?
+ Are you sure you want to hard reset supervisor <Tag
minimal>{resetSupervisorId}</Tag>?
</p>
- <p>Hard resetting a supervisor will lead to data loss or data
duplication.</p>
+ <p>Hard resetting a supervisor may lead to data loss or data
duplication.</p>
<p>
- The reason for using this operation is to recover from a state in
which the supervisor
- ceases operating due to missing offsets.
+ Use this operation to restore functionality when the supervisor
stops operating due to
+ missing offsets.
</p>
</AsyncActionDialog>
);
@@ -506,7 +597,7 @@ GROUP BY 1, 2`;
}}
>
<p>
- Are you sure you want to terminate supervisor
<Code>{terminateSupervisorId}</Code>?
+ Are you sure you want to terminate supervisor <Tag
minimal>{terminateSupervisorId}</Tag>?
</p>
<p>This action is not reversible.</p>
</AsyncActionDialog>
@@ -541,7 +632,7 @@ GROUP BY 1, 2`;
private renderSupervisorTable() {
const { goToTasks, filters, onFiltersChange } = this.props;
- const { supervisorsState, visibleColumns } = this.state;
+ const { supervisorsState, statsKey, visibleColumns } = this.state;
const supervisors = supervisorsState.data || [];
return (
@@ -554,6 +645,9 @@ GROUP BY 1, 2`;
filtered={filters}
onFilteredChange={onFiltersChange}
filterable
+ onFetchData={tableState => {
+ this.fetchData(tableState);
+ }}
defaultPageSize={SMALL_TABLE_PAGE_SIZE}
pageSizeOptions={SMALL_TABLE_PAGE_SIZE_OPTIONS}
showPagination={supervisors.length > SMALL_TABLE_PAGE_SIZE}
@@ -562,7 +656,7 @@ GROUP BY 1, 2`;
Header: twoLines('Supervisor ID', <i>(datasource)</i>),
id: 'supervisor_id',
accessor: 'supervisor_id',
- width: 300,
+ width: 280,
show: visibleColumns.shown('Supervisor ID'),
Cell: ({ value, original }) => (
<TableClickableCell
@@ -576,21 +670,21 @@ GROUP BY 1, 2`;
{
Header: 'Type',
accessor: 'type',
- width: 100,
+ width: 80,
Cell: this.renderSupervisorFilterableCell('type'),
show: visibleColumns.shown('Type'),
},
{
Header: 'Topic/Stream',
accessor: 'source',
- width: 300,
+ width: 200,
Cell: this.renderSupervisorFilterableCell('source'),
show: visibleColumns.shown('Topic/Stream'),
},
{
Header: 'Status',
- id: 'status',
- width: 250,
+ id: 'detailed_state',
+ width: 130,
accessor: 'detailed_state',
Cell: ({ value }) => (
<TableFilterableCell
@@ -607,28 +701,176 @@ GROUP BY 1, 2`;
),
show: visibleColumns.shown('Status'),
},
+ {
+ Header: 'Configured tasks',
+ id: 'configured_tasks',
+ width: 130,
+ accessor: 'spec',
+ filterable: false,
+ sortable: false,
+ className: 'padded',
+ Cell: ({ value }) => {
+ if (!value) return null;
+ const taskCount = deepGet(value, 'spec.ioConfig.taskCount');
+ const replicas = deepGet(value, 'spec.ioConfig.replicas');
+ if (typeof taskCount !== 'number' || typeof replicas !==
'number') return null;
+ return (
+ <div>
+ <div>{formatInteger(taskCount * replicas)}</div>
+ <div className="detail-line">
+ {replicas === 1
+ ? '(no replication)'
+ : `(${pluralIfNeeded(taskCount, 'task')} ×
${pluralIfNeeded(
+ replicas,
+ 'replica',
+ )})`}
+ </div>
+ </div>
+ );
+ },
+ show: visibleColumns.shown('Configured tasks'),
+ },
{
Header: 'Running tasks',
id: 'running_tasks',
width: 150,
- accessor: 'running_tasks',
+ accessor: 'status.payload',
filterable: false,
- Cell: ({ value, original }) => (
- <TableClickableCell
- onClick={() => goToTasks(original.supervisor_id,
`index_${original.type}`)}
- hoverIcon={IconNames.ARROW_TOP_RIGHT}
- title="Go to tasks"
+ sortable: false,
+ Cell: ({ value, original }) => {
+ if (original.suspended) return;
+ let label: string | JSX.Element;
+ const { activeTasks, publishingTasks } = value || {};
+ if (Array.isArray(activeTasks)) {
+ label = pluralIfNeeded(activeTasks.length, 'active task');
+ if (nonEmptyArray(publishingTasks)) {
+ label = (
+ <>
+ <div>{label}</div>
+ <div>{pluralIfNeeded(publishingTasks.length, 'publishing
task')}</div>
+ </>
+ );
+ }
+ } else {
+ label = 'n/a';
+ }
+ return (
+ <TableClickableCell
+ onClick={() => goToTasks(original.supervisor_id,
`index_${original.type}`)}
+ hoverIcon={IconNames.ARROW_TOP_RIGHT}
+ title="Go to tasks"
+ >
+ {label}
+ </TableClickableCell>
+ );
+ },
+ show: visibleColumns.shown('Running tasks'),
+ },
+ {
+ Header: 'Aggregate lag',
+ accessor: 'status.payload.aggregateLag',
+ width: 200,
+ filterable: false,
+ sortable: false,
+ className: 'padded',
+ show: visibleColumns.shown('Aggregate lag'),
+ Cell: ({ value }) => formatInteger(value),
+ },
+ {
+ Header: twoLines(
+ 'Stats',
+ <Popover2
+ position={Position.BOTTOM}
+ content={
+ <Menu>
+ {ROW_STATS_KEYS.map(k => (
+ <MenuItem
+ key={k}
+ icon={checkedCircleIcon(k === statsKey)}
+ text={getRowStatsKeyTitle(k)}
+ onClick={() => {
+ this.setState({ statsKey: k });
+ }}
+ />
+ ))}
+ </Menu>
+ }
>
- {typeof value === 'undefined'
- ? 'n/a'
- : value > 0
- ? pluralIfNeeded(value, 'running task')
- : original.suspended
- ? ''
- : `No running tasks`}
- </TableClickableCell>
+ <i className="title-button">
+ {getRowStatsKeyTitle(statsKey)} <Icon
icon={IconNames.CARET_DOWN} />
+ </i>
+ </Popover2>,
),
- show: visibleColumns.shown('Running tasks'),
+ id: 'stats',
+ width: 300,
+ filterable: false,
+ sortable: false,
+ className: 'padded',
+ accessor: 'stats',
+ Cell: ({ value, original }) => {
+ if (!value) return;
+ const activeTaskIds: string[] | undefined = deepGet(
+ original,
+ 'status.payload.activeTasks',
+ )?.map((t: SupervisorStatusTask) => t.id);
+ const c = getTotalSupervisorStats(value, statsKey,
activeTaskIds);
+ const seconds = getRowStatsKeySeconds(statsKey);
+ const totalLabel = `Total over ${statsKey}: `;
+ const bytes = c.processedBytes ? `
(${formatByteRate(c.processedBytes)})` : '';
+ return (
+ <div>
+ <div
+ title={`${totalLabel}${formatInteger(c.processed *
seconds)} (${formatBytes(
+ c.processedBytes * seconds,
+ )})`}
+ >{`Processed: ${formatRate(c.processed)}${bytes}`}</div>
+ {Boolean(c.processedWithError) && (
+ <div
+ className="warning-line"
+
title={`${totalLabel}${formatInteger(c.processedWithError * seconds)}`}
+ >
+ Processed with error: {formatRate(c.processedWithError)}
+ </div>
+ )}
+ {Boolean(c.thrownAway) && (
+ <div
+ className="warning-line"
+ title={`${totalLabel}${formatInteger(c.thrownAway *
seconds)}`}
+ >
+ Thrown away: {formatRate(c.thrownAway)}
+ </div>
+ )}
+ {Boolean(c.unparseable) && (
+ <div
+ className="warning-line"
+ title={`${totalLabel}${formatInteger(c.unparseable *
seconds)}`}
+ >
+ Unparseable: {formatRate(c.unparseable)}
+ </div>
+ )}
+ </div>
+ );
+ },
+ show: visibleColumns.shown('Stats'),
+ },
+ {
+ Header: 'Recent errors',
+ accessor: 'status.payload.recentErrors',
+ width: 150,
+ filterable: false,
+ sortable: false,
+ show: visibleColumns.shown('Recent errors'),
+ Cell: ({ value, original }) => {
+ return (
+ <TableClickableCell
+ onClick={() => this.onSupervisorDetail(original)}
+ hoverIcon={IconNames.SEARCH_TEMPLATE}
+ title="See errors"
+ >
+ {pluralIfNeeded(value?.length, 'error')}
+ </TableClickableCell>
+ );
+ },
},
{
Header: ACTION_COLUMN_LABEL,
@@ -636,6 +878,7 @@ GROUP BY 1, 2`;
accessor: 'supervisor_id',
width: ACTION_COLUMN_WIDTH,
filterable: false,
+ sortable: false,
Cell: row => {
const id = row.value;
const type = row.original.type;
@@ -648,7 +891,6 @@ GROUP BY 1, 2`;
/>
);
},
- show: visibleColumns.shown(ACTION_COLUMN_LABEL),
},
]}
/>
@@ -657,6 +899,7 @@ GROUP BY 1, 2`;
renderBulkSupervisorActions() {
const { capabilities, goToQuery } = this.props;
+ const lastSupervisorQuery =
this.supervisorQueryManager.getLastIntermediateQuery();
return (
<>
@@ -665,7 +908,7 @@ GROUP BY 1, 2`;
<MenuItem
icon={IconNames.APPLICATION}
text="View SQL query for table"
- onClick={() => goToQuery({ queryString:
SupervisorsView.SUPERVISOR_SQL })}
+ onClick={() => goToQuery({ queryString: lastSupervisorQuery })}
/>
)}
<MenuItem
@@ -796,7 +1039,7 @@ GROUP BY 1, 2`;
/>
{this.renderBulkSupervisorActions()}
<TableColumnSelector
- columns={supervisorTableColumns}
+ columns={SUPERVISOR_TABLE_COLUMNS}
onChange={column =>
this.setState(prevState => ({
visibleColumns: prevState.visibleColumns.toggle(column),
diff --git
a/web-console/src/views/tasks-view/__snapshots__/tasks-view.spec.tsx.snap
b/web-console/src/views/tasks-view/__snapshots__/tasks-view.spec.tsx.snap
index 93779dfc0bf..178b50d9f57 100644
--- a/web-console/src/views/tasks-view/__snapshots__/tasks-view.spec.tsx.snap
+++ b/web-console/src/views/tasks-view/__snapshots__/tasks-view.spec.tsx.snap
@@ -81,7 +81,6 @@ exports[`TasksView matches snapshot 1`] = `
"Created time",
"Duration",
"Location",
- "Actions",
]
}
onChange={[Function]}
@@ -217,7 +216,7 @@ exports[`TasksView matches snapshot 1`] = `
"accessor": "task_id",
"filterable": false,
"id": "actions",
- "show": true,
+ "sortable": false,
"width": 70,
},
]
diff --git a/web-console/src/views/tasks-view/tasks-view.tsx
b/web-console/src/views/tasks-view/tasks-view.tsx
index 795c9908412..49a66b71cb1 100644
--- a/web-console/src/views/tasks-view/tasks-view.tsx
+++ b/web-console/src/views/tasks-view/tasks-view.tsx
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import { Button, ButtonGroup, Intent, Label, MenuItem } from
'@blueprintjs/core';
+import { Button, ButtonGroup, Intent, Label, MenuItem, Tag } from
'@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import React from 'react';
import type { Filter } from 'react-table';
@@ -65,7 +65,6 @@ const taskTableColumns: string[] = [
'Created time',
'Duration',
'Location',
- ACTION_COLUMN_LABEL,
];
interface TaskQueryResultRow {
@@ -310,7 +309,9 @@ ORDER BY
this.taskQueryManager.rerunLastQuery();
}}
>
- <p>{`Are you sure you want to kill task '${killTaskId}'?`}</p>
+ <p>
+ Are you sure you want to kill task <Tag minimal>{killTaskId}</Tag>?
+ </p>
</AsyncActionDialog>
);
}
@@ -494,6 +495,7 @@ ORDER BY
accessor: 'task_id',
width: ACTION_COLUMN_WIDTH,
filterable: false,
+ sortable: false,
Cell: row => {
if (row.aggregated) return '';
const id = row.value;
@@ -507,7 +509,6 @@ ORDER BY
);
},
Aggregated: () => '',
- show: visibleColumns.shown(ACTION_COLUMN_LABEL),
},
]}
/>
diff --git
a/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx
b/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx
index bf6bb165c85..9c53c375760 100644
--- a/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx
+++ b/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx
@@ -53,10 +53,15 @@ export const MaxTasksButton = function
MaxTasksButton(props: MaxTasksButtonProps
const maxNumTasks = getMaxNumTasks(queryContext);
const taskAssigment = getTaskAssigment(queryContext);
- const fullClusterCapacity = `${clusterCapacity} (full cluster capacity)`;
+ const fullClusterCapacity =
+ typeof clusterCapacity === 'number'
+ ? `${formatInteger(clusterCapacity)} (full cluster capacity)`
+ : undefined;
+
const shownMaxNumTaskOptions = clusterCapacity
? MAX_NUM_TASK_OPTIONS.filter(_ => _ <= clusterCapacity)
: MAX_NUM_TASK_OPTIONS;
+
return (
<>
<Popover2
@@ -65,7 +70,7 @@ export const MaxTasksButton = function MaxTasksButton(props:
MaxTasksButtonProps
content={
<Menu>
<MenuDivider title="Maximum number of tasks to launch" />
- {Boolean(clusterCapacity) && (
+ {Boolean(fullClusterCapacity) && (
<MenuItem
icon={tickIcon(typeof maxNumTasks === 'undefined')}
text={fullClusterCapacity}
@@ -115,7 +120,7 @@ export const MaxTasksButton = function
MaxTasksButton(props: MaxTasksButtonProps
? clusterCapacity
? fullClusterCapacity
: 2
- : maxNumTasks
+ : formatInteger(maxNumTasks)
}`}
rightIcon={IconNames.CARET_DOWN}
/>
diff --git a/web-console/src/views/workbench-view/workbench-view.tsx
b/web-console/src/views/workbench-view/workbench-view.tsx
index 76699f52a08..c251a50f19a 100644
--- a/web-console/src/views/workbench-view/workbench-view.tsx
+++ b/web-console/src/views/workbench-view/workbench-view.tsx
@@ -629,6 +629,7 @@ export class WorkbenchView extends
React.PureComponent<WorkbenchViewProps, Workb
this.props;
const { columnMetadataState } = this.state;
const currentTabEntry = this.getCurrentTabEntry();
+ const effectiveEngine = currentTabEntry.query.getEffectiveEngine();
return (
<div className="center-panel">
@@ -650,14 +651,15 @@ export class WorkbenchView extends
React.PureComponent<WorkbenchViewProps, Workb
goToTask={goToTask}
runMoreMenu={
<Menu>
- {allowExplain && (
- <MenuItem
- icon={IconNames.CLEAN}
- text="Explain SQL query"
- onClick={this.openExplainDialog}
- />
- )}
- {currentTabEntry.query.getEffectiveEngine() !== 'sql-msq-task'
&& (
+ {allowExplain &&
+ (effectiveEngine === 'sql-native' || effectiveEngine ===
'sql-msq-task') && (
+ <MenuItem
+ icon={IconNames.CLEAN}
+ text="Explain SQL query"
+ onClick={this.openExplainDialog}
+ />
+ )}
+ {effectiveEngine !== 'sql-msq-task' && (
<MenuItem
icon={IconNames.HISTORY}
text="Query history"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
