This is an automated email from the ASF dual-hosted git repository. vogievetsky pushed a commit to branch segment_timeline2 in repository https://gitbox.apache.org/repos/asf/druid.git
commit 952b0fb2608fabcaf96fc6ab38d3984aa92f6e43 Author: Vadim Ogievetsky <[email protected]> AuthorDate: Mon Nov 4 09:42:08 2024 -0800 api --- .../segment-timeline/segment-bar-chart.tsx | 9 +- .../segment-timeline/segment-timeline.tsx | 4 +- .../supervisor-history-panel.tsx | 13 ++- .../compaction-history-dialog.tsx | 7 +- .../coordinator-dynamic-config-dialog.tsx | 7 +- .../overlord-dynamic-config-dialog.tsx | 7 +- .../dialogs/retention-dialog/retention-dialog.tsx | 15 +-- .../src/dialogs/status-dialog/status-dialog.tsx | 3 +- .../supervisor-reset-offsets-dialog.tsx | 11 +- web-console/src/utils/druid-query.ts | 6 + web-console/src/utils/table-helpers.ts | 11 ++ .../views/datasources-view/datasources-view.tsx | 18 +-- .../datasources-card/datasources-card.tsx | 10 +- .../home-view/segments-card/segments-card.tsx | 7 +- .../home-view/services-card/services-card.tsx | 13 +-- .../supervisors-card/supervisors-card.tsx | 15 ++- .../src/views/home-view/tasks-card/tasks-card.tsx | 17 ++- .../src/views/load-data-view/load-data-view.tsx | 4 +- .../src/views/lookups-view/lookups-view.tsx | 9 +- .../src/views/segments-view/segments-view.tsx | 124 +++++++++------------ .../src/views/services-view/services-view.tsx | 44 ++++---- .../views/supervisors-view/supervisors-view.tsx | 37 +++--- web-console/src/views/tasks-view/tasks-view.tsx | 32 +++--- 23 files changed, 201 insertions(+), 222 deletions(-) diff --git a/web-console/src/components/segment-timeline/segment-bar-chart.tsx b/web-console/src/components/segment-timeline/segment-bar-chart.tsx index 17f2866abee..bcbe645c58e 100644 --- a/web-console/src/components/segment-timeline/segment-bar-chart.tsx +++ b/web-console/src/components/segment-timeline/segment-bar-chart.tsx @@ -23,7 +23,7 @@ import { useMemo } from 'react'; import type { Capabilities } from '../../helpers'; import { useQueryManager } from '../../hooks'; import { Api } from '../../singletons'; -import { Duration, filterMap, queryDruidSql, TZ_UTC } from '../../utils'; +import { Duration, filterMap, getApiArray, queryDruidSql, TZ_UTC } from '../../utils'; import type { Stage } from '../../utils/stage'; import { Loader } from '../loader/loader'; @@ -82,9 +82,10 @@ export const SegmentBarChart = function SegmentBarChart(props: SegmentBarChartPr }; }); // This trimming should ideally be pushed into the SQL query but at the time of this writing queries on the sys.* tables do not allow substring } else { - const datasources: string[] = ( - await Api.instance.get(`/druid/coordinator/v1/datasources`, { cancelToken }) - ).data; + const datasources = await getApiArray<string>( + `/druid/coordinator/v1/datasources`, + cancelToken, + ); return ( await Promise.all( diff --git a/web-console/src/components/segment-timeline/segment-timeline.tsx b/web-console/src/components/segment-timeline/segment-timeline.tsx index 5fbc2587ea2..2a683b011ea 100644 --- a/web-console/src/components/segment-timeline/segment-timeline.tsx +++ b/web-console/src/components/segment-timeline/segment-timeline.tsx @@ -34,10 +34,10 @@ import { useState } from 'react'; import type { Capabilities } from '../../helpers'; import { useQueryManager } from '../../hooks'; -import { Api } from '../../singletons'; import { day, Duration, + getApiArray, isNonNullRange, localToUtcDateRange, prettyFormatIsoDate, @@ -95,7 +95,7 @@ export const SegmentTimeline = function SegmentTimeline(props: SegmentTimelinePr return tables.map(d => d.TABLE_NAME); } else { - return (await Api.instance.get(`/druid/coordinator/v1/datasources`, { cancelToken })).data; + return await getApiArray(`/druid/coordinator/v1/datasources`, cancelToken); } }, }); diff --git a/web-console/src/components/supervisor-history-panel/supervisor-history-panel.tsx b/web-console/src/components/supervisor-history-panel/supervisor-history-panel.tsx index 74cb55682f3..a7fa38fd445 100644 --- a/web-console/src/components/supervisor-history-panel/supervisor-history-panel.tsx +++ b/web-console/src/components/supervisor-history-panel/supervisor-history-panel.tsx @@ -25,7 +25,7 @@ import type { IngestionSpec } from '../../druid-models'; import { cleanSpec } from '../../druid-models'; import { useQueryManager } from '../../hooks'; import { Api } from '../../singletons'; -import { deepSet } from '../../utils'; +import { deepSet, getApiArray } from '../../utils'; import { Loader } from '../loader/loader'; import { ShowValue } from '../show-value/show-value'; @@ -49,11 +49,12 @@ export const SupervisorHistoryPanel = React.memo(function SupervisorHistoryPanel const [historyState] = useQueryManager<string, SupervisorHistoryEntry[]>({ initQuery: supervisorId, processQuery: async (supervisorId, cancelToken) => { - const resp = await Api.instance.get( - `/druid/indexer/v1/supervisor/${Api.encodePath(supervisorId)}/history`, - { cancelToken }, - ); - return resp.data.map((vs: SupervisorHistoryEntry) => deepSet(vs, 'spec', cleanSpec(vs.spec))); + return ( + await getApiArray<SupervisorHistoryEntry>( + `/druid/indexer/v1/supervisor/${Api.encodePath(supervisorId)}/history`, + cancelToken, + ) + ).map(vs => deepSet(vs, 'spec', cleanSpec(vs.spec))); }, }); diff --git a/web-console/src/dialogs/compaction-history-dialog/compaction-history-dialog.tsx b/web-console/src/dialogs/compaction-history-dialog/compaction-history-dialog.tsx index 9e19e043c71..8ee0ffdbfbc 100644 --- a/web-console/src/dialogs/compaction-history-dialog/compaction-history-dialog.tsx +++ b/web-console/src/dialogs/compaction-history-dialog/compaction-history-dialog.tsx @@ -24,7 +24,7 @@ import { Loader, ShowValue } from '../../components'; import type { CompactionConfig } from '../../druid-models'; import { useQueryManager } from '../../hooks'; import { Api } from '../../singletons'; -import { formatInteger, formatPercent } from '../../utils'; +import { formatInteger, formatPercent, getApiArray } from '../../utils'; import { DiffDialog } from '../diff-dialog/diff-dialog'; import './compaction-history-dialog.scss'; @@ -65,11 +65,10 @@ export const CompactionHistoryDialog = React.memo(function CompactionHistoryDial initQuery: datasource, processQuery: async (datasource, cancelToken) => { try { - const resp = await Api.instance.get( + return await getApiArray<CompactionHistoryEntry>( `/druid/coordinator/v1/config/compaction/${Api.encodePath(datasource)}/history?count=20`, - { cancelToken }, + cancelToken, ); - return resp.data; } catch (e) { if (e.response?.status === 404) return []; throw e; diff --git a/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx b/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx index ec964f5507e..ab4fed8ae1a 100644 --- a/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx +++ b/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx @@ -27,7 +27,7 @@ import { COORDINATOR_DYNAMIC_CONFIG_FIELDS } from '../../druid-models'; import { useQueryManager } from '../../hooks'; import { getLink } from '../../links'; import { Api, AppToaster } from '../../singletons'; -import { getDruidErrorMessage } from '../../utils'; +import { getApiArray, getDruidErrorMessage } from '../../utils'; import { SnitchDialog } from '..'; import './coordinator-dynamic-config-dialog.scss'; @@ -47,10 +47,7 @@ export const CoordinatorDynamicConfigDialog = React.memo(function CoordinatorDyn const [historyRecordsState] = useQueryManager<null, any[]>({ initQuery: null, processQuery: async (_, cancelToken) => { - const historyResp = await Api.instance.get(`/druid/coordinator/v1/config/history?count=100`, { - cancelToken, - }); - return historyResp.data; + return await getApiArray(`/druid/coordinator/v1/config/history?count=100`, cancelToken); }, }); diff --git a/web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx b/web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx index ba30118b0a3..5b1233c6384 100644 --- a/web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx +++ b/web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx @@ -27,7 +27,7 @@ import { OVERLORD_DYNAMIC_CONFIG_FIELDS } from '../../druid-models'; import { useQueryManager } from '../../hooks'; import { getLink } from '../../links'; import { Api, AppToaster } from '../../singletons'; -import { getDruidErrorMessage } from '../../utils'; +import { getApiArray, getDruidErrorMessage } from '../../utils'; import { SnitchDialog } from '..'; import './overlord-dynamic-config-dialog.scss'; @@ -47,10 +47,7 @@ export const OverlordDynamicConfigDialog = React.memo(function OverlordDynamicCo const [historyRecordsState] = useQueryManager<null, any[]>({ initQuery: null, processQuery: async (_, cancelToken) => { - const historyResp = await Api.instance.get(`/druid/indexer/v1/worker/history?count=100`, { - cancelToken, - }); - return historyResp.data; + return await getApiArray(`/druid/indexer/v1/worker/history?count=100`, cancelToken); }, }); diff --git a/web-console/src/dialogs/retention-dialog/retention-dialog.tsx b/web-console/src/dialogs/retention-dialog/retention-dialog.tsx index 5ee4d51a3a5..e656a4cbdff 100644 --- a/web-console/src/dialogs/retention-dialog/retention-dialog.tsx +++ b/web-console/src/dialogs/retention-dialog/retention-dialog.tsx @@ -27,7 +27,7 @@ import type { Capabilities } from '../../helpers'; import { useQueryManager } from '../../hooks'; import { getLink } from '../../links'; import { Api } from '../../singletons'; -import { filterMap, queryDruidSql, swapElements } from '../../utils'; +import { filterMap, getApiArray, queryDruidSql, swapElements } from '../../utils'; import { SnitchDialog } from '..'; import './retention-dialog.scss'; @@ -67,11 +67,9 @@ ORDER BY 1`, return sqlResp.map(d => d.tier); } else if (capabilities.hasCoordinatorAccess()) { - const allServiceResp = await Api.instance.get('/druid/coordinator/v1/servers?simple', { - cancelToken, - }); - return filterMap(allServiceResp.data, (s: any) => - s.type === 'historical' ? s.tier : undefined, + return filterMap( + await getApiArray('/druid/coordinator/v1/servers?simple', cancelToken), + (s: any) => (s.type === 'historical' ? s.tier : undefined), ); } else { throw new Error(`must have sql or coordinator access`); @@ -84,11 +82,10 @@ ORDER BY 1`, const [historyQueryState] = useQueryManager<string, any[]>({ initQuery: props.datasource, processQuery: async (datasource, cancelToken) => { - const historyResp = await Api.instance.get( + return await getApiArray( `/druid/coordinator/v1/rules/${Api.encodePath(datasource)}/history?count=200`, - { cancelToken }, + cancelToken, ); - return historyResp.data; }, }); diff --git a/web-console/src/dialogs/status-dialog/status-dialog.tsx b/web-console/src/dialogs/status-dialog/status-dialog.tsx index 311f3e05664..672fc40910f 100644 --- a/web-console/src/dialogs/status-dialog/status-dialog.tsx +++ b/web-console/src/dialogs/status-dialog/status-dialog.tsx @@ -50,8 +50,7 @@ export const StatusDialog = React.memo(function StatusDialog(props: StatusDialog const [responseState] = useQueryManager<null, StatusResponse>({ initQuery: null, processQuery: async (_, cancelToken) => { - const resp = await Api.instance.get(`/status`, { cancelToken }); - return resp.data; + return (await Api.instance.get(`/status`, { cancelToken })).data; }, }); diff --git a/web-console/src/dialogs/supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog.tsx b/web-console/src/dialogs/supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog.tsx index 009d8326060..d10fc00eb5c 100644 --- a/web-console/src/dialogs/supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog.tsx +++ b/web-console/src/dialogs/supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog.tsx @@ -106,11 +106,12 @@ export const SupervisorResetOffsetsDialog = React.memo(function SupervisorResetO const [statusResp] = useQueryManager<string, SupervisorStatus>({ initQuery: supervisorId, processQuery: async (supervisorId, cancelToken) => { - const statusResp = await Api.instance.get( - `/druid/indexer/v1/supervisor/${Api.encodePath(supervisorId)}/status`, - { cancelToken }, - ); - return statusResp.data; + return ( + await Api.instance.get( + `/druid/indexer/v1/supervisor/${Api.encodePath(supervisorId)}/status`, + { cancelToken }, + ) + ).data; }, }); diff --git a/web-console/src/utils/druid-query.ts b/web-console/src/utils/druid-query.ts index 8102db89ca3..9e33b01a4e3 100644 --- a/web-console/src/utils/druid-query.ts +++ b/web-console/src/utils/druid-query.ts @@ -358,6 +358,12 @@ export async function queryDruidSqlDart<T = any>( return sqlResultResp.data; } +export async function getApiArray<T = any>(url: string, cancelToken?: CancelToken): Promise<T[]> { + const result = (await Api.instance.get(url, { cancelToken })).data; + if (!Array.isArray(result)) throw new Error('unexpected result'); + return result; +} + export interface QueryExplanation { query: any; signature: { name: string; type: string }[]; diff --git a/web-console/src/utils/table-helpers.ts b/web-console/src/utils/table-helpers.ts index 117b34f2d1a..b2073944e46 100644 --- a/web-console/src/utils/table-helpers.ts +++ b/web-console/src/utils/table-helpers.ts @@ -18,6 +18,7 @@ import type { QueryResult, SqlExpression } from '@druid-toolkit/query'; import { C } from '@druid-toolkit/query'; +import { ascending, descending, sort } from 'd3-array'; import type { Filter, SortingRule } from 'react-table'; import { filterMap, formatNumber, isNumberLike, oneOf } from './general'; @@ -78,3 +79,13 @@ export function sortedToOrderByClause(sorted: SortingRule[]): string | undefined if (!sorted.length) return; return 'ORDER BY ' + sorted.map(sort => `${C(sort.id)} ${sort.desc ? 'DESC' : 'ASC'}`).join(', '); } + +export function applySorting(xs: any[], sorted: SortingRule[]): any[] { + const firstSortingRule = sorted[0]; + if (!firstSortingRule) return xs; + const { id, desc } = firstSortingRule; + return sort( + xs, + desc ? (d1, d2) => descending(d1[id], d2[id]) : (d1, d2) => ascending(d1[id], d2[id]), + ); +} diff --git a/web-console/src/views/datasources-view/datasources-view.tsx b/web-console/src/views/datasources-view/datasources-view.tsx index 3d0e6799af6..bfa2262dad1 100644 --- a/web-console/src/views/datasources-view/datasources-view.tsx +++ b/web-console/src/views/datasources-view/datasources-view.tsx @@ -74,6 +74,7 @@ import { formatInteger, formatMillions, formatPercent, + getApiArray, getDruidErrorMessage, groupByAsMap, hasPopoverOpen, @@ -441,15 +442,15 @@ GROUP BY 1, 2`; setIntermediateQuery(query); datasources = await queryDruidSql({ query }, cancelToken); } else if (capabilities.hasCoordinatorAccess()) { - const datasourcesResp = await Api.instance.get( + const datasourcesResp = await getApiArray( '/druid/coordinator/v1/datasources?simple', - { cancelToken }, + cancelToken, ); const loadstatusResp = await Api.instance.get('/druid/coordinator/v1/loadstatus?simple', { cancelToken, }); const loadstatus = loadstatusResp.data; - datasources = datasourcesResp.data.map((d: any): DatasourceQueryResultRow => { + datasources = datasourcesResp.map((d: any): DatasourceQueryResultRow => { const totalDataSize = deepGet(d, 'properties.segments.size') || -1; const segmentsToLoad = Number(loadstatus[d.name] || 0); const availableSegments = Number(deepGet(d, 'properties.segments.count')); @@ -527,9 +528,10 @@ GROUP BY 1, 2`; if (capabilities.hasOverlordAccess()) { auxiliaryQueries.push(async (datasourcesAndDefaultRules, cancelToken) => { try { - const taskList = ( - await Api.instance.get(`/druid/indexer/v1/tasks?state=running`, { cancelToken }) - ).data; + const taskList = await getApiArray( + `/druid/indexer/v1/tasks?state=running`, + cancelToken, + ); const runningTasksByDatasource = groupByAsMap( taskList, @@ -568,10 +570,10 @@ GROUP BY 1, 2`; if (showUnused) { try { unused = ( - await Api.instance.get<string[]>( + await getApiArray<string>( '/druid/coordinator/v1/metadata/datasources?includeUnused', ) - ).data.filter(d => !seen[d]); + ).filter(d => !seen[d]); } catch { AppToaster.show({ icon: IconNames.ERROR, diff --git a/web-console/src/views/home-view/datasources-card/datasources-card.tsx b/web-console/src/views/home-view/datasources-card/datasources-card.tsx index 811187117e7..d2a219776c5 100644 --- a/web-console/src/views/home-view/datasources-card/datasources-card.tsx +++ b/web-console/src/views/home-view/datasources-card/datasources-card.tsx @@ -21,8 +21,7 @@ import React from 'react'; import type { Capabilities } from '../../../helpers'; import { useQueryManager } from '../../../hooks'; -import { Api } from '../../../singletons'; -import { pluralIfNeeded, queryDruidSql } from '../../../utils'; +import { getApiArray, pluralIfNeeded, queryDruidSql } from '../../../utils'; import { HomeViewCard } from '../home-view-card/home-view-card'; export interface DatasourcesCardProps { @@ -33,7 +32,7 @@ export const DatasourcesCard = React.memo(function DatasourcesCard(props: Dataso const [datasourceCountState] = useQueryManager<Capabilities, number>({ initQuery: props.capabilities, processQuery: async (capabilities, cancelToken) => { - let datasources: any[]; + let datasources: string[]; if (capabilities.hasSql()) { datasources = await queryDruidSql( { @@ -42,10 +41,7 @@ export const DatasourcesCard = React.memo(function DatasourcesCard(props: Dataso cancelToken, ); } else if (capabilities.hasCoordinatorAccess()) { - const datasourcesResp = await Api.instance.get('/druid/coordinator/v1/datasources', { - cancelToken, - }); - datasources = datasourcesResp.data; + datasources = await getApiArray<string>('/druid/coordinator/v1/datasources', cancelToken); } else { throw new Error(`must have SQL or coordinator access`); } diff --git a/web-console/src/views/home-view/segments-card/segments-card.tsx b/web-console/src/views/home-view/segments-card/segments-card.tsx index d5d157a523b..6a86fe5f18e 100644 --- a/web-console/src/views/home-view/segments-card/segments-card.tsx +++ b/web-console/src/views/home-view/segments-card/segments-card.tsx @@ -23,7 +23,7 @@ import React from 'react'; import type { Capabilities } from '../../../helpers'; import { useQueryManager } from '../../../hooks'; import { Api } from '../../../singletons'; -import { deepGet, pluralIfNeeded, queryDruidSql } from '../../../utils'; +import { deepGet, getApiArray, pluralIfNeeded, queryDruidSql } from '../../../utils'; import { HomeViewCard } from '../home-view-card/home-view-card'; export interface SegmentCounts { @@ -62,11 +62,10 @@ WHERE is_active = 1`, const loadstatus = loadstatusResp.data; const unavailableSegmentNum = sum(Object.keys(loadstatus), key => loadstatus[key]); - const datasourcesMetaResp = await Api.instance.get( + const datasourcesMeta = await getApiArray( '/druid/coordinator/v1/datasources?simple', - { cancelToken }, + cancelToken, ); - const datasourcesMeta = datasourcesMetaResp.data; const availableSegmentNum = sum(datasourcesMeta, (curr: any) => deepGet(curr, 'properties.segments.count'), ); diff --git a/web-console/src/views/home-view/services-card/services-card.tsx b/web-console/src/views/home-view/services-card/services-card.tsx index 18a66b83218..a04680237a4 100644 --- a/web-console/src/views/home-view/services-card/services-card.tsx +++ b/web-console/src/views/home-view/services-card/services-card.tsx @@ -22,8 +22,7 @@ import React from 'react'; import { PluralPairIfNeeded } from '../../../components'; import type { Capabilities } from '../../../helpers'; import { useQueryManager } from '../../../hooks'; -import { Api } from '../../../singletons'; -import { lookupBy, queryDruidSql } from '../../../utils'; +import { getApiArray, lookupBy, queryDruidSql } from '../../../utils'; import { HomeViewCard } from '../home-view-card/home-view-card'; export interface ServiceCounts { @@ -45,10 +44,10 @@ export const ServicesCard = React.memo(function ServicesCard(props: ServicesCard const [serviceCountState] = useQueryManager<Capabilities, ServiceCounts>({ processQuery: async (capabilities, cancelToken) => { if (capabilities.hasSql()) { - const serviceCountsFromQuery: { + const serviceCountsFromQuery = await queryDruidSql<{ service_type: string; count: number; - }[] = await queryDruidSql( + }>( { query: `SELECT server_type AS "service_type", COUNT(*) as "count" FROM sys.servers GROUP BY 1`, }, @@ -60,12 +59,10 @@ export const ServicesCard = React.memo(function ServicesCard(props: ServicesCard x => x.count, ); } else if (capabilities.hasCoordinatorAccess()) { - const services = ( - await Api.instance.get('/druid/coordinator/v1/servers?simple', { cancelToken }) - ).data; + const services = await getApiArray('/druid/coordinator/v1/servers?simple', cancelToken); const middleManager = capabilities.hasOverlordAccess() - ? (await Api.instance.get('/druid/indexer/v1/workers', { cancelToken })).data + ? await getApiArray('/druid/indexer/v1/workers', cancelToken) : []; return { diff --git a/web-console/src/views/home-view/supervisors-card/supervisors-card.tsx b/web-console/src/views/home-view/supervisors-card/supervisors-card.tsx index ab63e0205ac..dca25e32d36 100644 --- a/web-console/src/views/home-view/supervisors-card/supervisors-card.tsx +++ b/web-console/src/views/home-view/supervisors-card/supervisors-card.tsx @@ -19,10 +19,10 @@ import { IconNames } from '@blueprintjs/icons'; import React from 'react'; +import type { IngestionSpec } from '../../../druid-models'; import type { Capabilities } from '../../../helpers'; import { useQueryManager } from '../../../hooks'; -import { Api } from '../../../singletons'; -import { pluralIfNeeded, queryDruidSql } from '../../../utils'; +import { getApiArray, partition, pluralIfNeeded, queryDruidSql } from '../../../utils'; import { HomeViewCard } from '../home-view-card/home-view-card'; export interface SupervisorCounts { @@ -50,11 +50,14 @@ FROM sys.supervisors`, ) )[0]; } else if (capabilities.hasOverlordAccess()) { - const resp = await Api.instance.get('/druid/indexer/v1/supervisor?full', { cancelToken }); - const data = resp.data; + const supervisors = await getApiArray<{ spec: IngestionSpec }>( + '/druid/indexer/v1/supervisor?full', + cancelToken, + ); + const [running, suspended] = partition(supervisors, d => !d.spec.suspended); return { - running: data.filter((d: any) => d.spec.suspended === false).length, - suspended: data.filter((d: any) => d.spec.suspended === true).length, + running: running.length, + suspended: suspended.length, }; } else { throw new Error(`must have SQL or overlord access`); diff --git a/web-console/src/views/home-view/tasks-card/tasks-card.tsx b/web-console/src/views/home-view/tasks-card/tasks-card.tsx index fa91b8a141e..901a4e15581 100644 --- a/web-console/src/views/home-view/tasks-card/tasks-card.tsx +++ b/web-console/src/views/home-view/tasks-card/tasks-card.tsx @@ -25,8 +25,7 @@ import type { CapacityInfo } from '../../../druid-models'; import type { Capabilities } from '../../../helpers'; import { getClusterCapacity } from '../../../helpers'; import { useQueryManager } from '../../../hooks'; -import { Api } from '../../../singletons'; -import { lookupBy, pluralIfNeeded, queryDruidSql } from '../../../utils'; +import { getApiArray, groupByAsMap, lookupBy, pluralIfNeeded, queryDruidSql } from '../../../utils'; import { HomeViewCard } from '../home-view-card/home-view-card'; function getTaskStatus(d: any) { @@ -62,14 +61,12 @@ GROUP BY 1`, x => x.count, ); } else if (capabilities.hasOverlordAccess()) { - const tasks: any[] = (await Api.instance.get('/druid/indexer/v1/tasks', { cancelToken })).data; - return { - success: tasks.filter(d => getTaskStatus(d) === 'SUCCESS').length, - failed: tasks.filter(d => getTaskStatus(d) === 'FAILED').length, - running: tasks.filter(d => getTaskStatus(d) === 'RUNNING').length, - pending: tasks.filter(d => getTaskStatus(d) === 'PENDING').length, - waiting: tasks.filter(d => getTaskStatus(d) === 'WAITING').length, - }; + const tasks: any[] = await getApiArray('/druid/indexer/v1/tasks', cancelToken); + return groupByAsMap( + tasks, + d => getTaskStatus(d).toLowerCase(), + xs => xs.length, + ); } else { throw new Error(`must have SQL or overlord access`); } diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx index b4bd0d841b6..3767a9852e9 100644 --- a/web-console/src/views/load-data-view/load-data-view.tsx +++ b/web-console/src/views/load-data-view/load-data-view.tsx @@ -157,6 +157,7 @@ import { EMPTY_ARRAY, EMPTY_OBJECT, filterMap, + getApiArray, getDruidErrorMessage, localStorageGetJson, LocalStorageKeys, @@ -3559,8 +3560,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat let existingDatasources: string[]; try { - existingDatasources = (await Api.instance.get<string[]>('/druid/coordinator/v1/datasources')) - .data; + existingDatasources = await getApiArray<string>('/druid/coordinator/v1/datasources'); } catch { return; } diff --git a/web-console/src/views/lookups-view/lookups-view.tsx b/web-console/src/views/lookups-view/lookups-view.tsx index 2fbfb70703e..f9d188078a7 100644 --- a/web-console/src/views/lookups-view/lookups-view.tsx +++ b/web-console/src/views/lookups-view/lookups-view.tsx @@ -41,6 +41,7 @@ import { STANDARD_TABLE_PAGE_SIZE, STANDARD_TABLE_PAGE_SIZE_OPTIONS } from '../. import { Api, AppToaster } from '../../singletons'; import { deepGet, + getApiArray, getDruidErrorMessage, hasPopoverOpen, isLookupsUninitialized, @@ -124,14 +125,12 @@ export class LookupsView extends React.PureComponent<LookupsViewProps, LookupsVi this.lookupsQueryManager = new QueryManager({ processQuery: async (_, cancelToken) => { - const tiersResp = await Api.instance.get( + const tiersResp = await getApiArray( '/druid/coordinator/v1/lookups/config?discover=true', - { cancelToken }, + cancelToken, ); const tiers = - tiersResp.data && tiersResp.data.length > 0 - ? tiersResp.data.sort(tierNameCompare) - : [DEFAULT_LOOKUP_TIER]; + tiersResp.length > 0 ? tiersResp.sort(tierNameCompare) : [DEFAULT_LOOKUP_TIER]; const lookupResp = await Api.instance.get('/druid/coordinator/v1/lookups/config/all', { cancelToken, diff --git a/web-console/src/views/segments-view/segments-view.tsx b/web-console/src/views/segments-view/segments-view.tsx index 8f9aeb5abd4..af93f65b8fb 100644 --- a/web-console/src/views/segments-view/segments-view.tsx +++ b/web-console/src/views/segments-view/segments-view.tsx @@ -58,12 +58,13 @@ import { import { Api } from '../../singletons'; import type { NumberLike, TableState } from '../../utils'; import { + applySorting, compact, countBy, - deepGet, filterMap, formatBytes, formatInteger, + getApiArray, hasPopoverOpen, isNumberLikeNaN, LocalStorageBackedVisibility, @@ -133,6 +134,9 @@ const TABLE_COLUMNS_BY_MODE: Record<CapabilitiesMode, TableColumnSelectorColumn[ 'Shard spec', 'Partition', 'Size', + 'Replication factor', + 'Is realtime', + 'Is overshadowed', ], }; @@ -365,75 +369,60 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment return result as SegmentQueryResultRow[]; } else if (capabilities.hasCoordinatorAccess()) { - let datasourceList: string[] = ( - await Api.instance.get('/druid/coordinator/v1/metadata/datasources', { cancelToken }) - ).data; - + let datasourceList: string[] = []; const datasourceFilter = filtered.find(({ id }) => id === 'datasource'); if (datasourceFilter) { - datasourceList = datasourceList.filter(datasource => + datasourceList = ( + await getApiArray('/druid/coordinator/v1/metadata/datasources', cancelToken) + ).filter((datasource: string) => booleanCustomTableFilter(datasourceFilter, datasource), ); } - if (sorted.length && sorted[0].id === 'datasource') { - datasourceList.sort( - sorted[0].desc ? (d1, d2) => d2.localeCompare(d1) : (d1, d2) => d1.localeCompare(d2), - ); - } - - const maxResults = (page + 1) * pageSize; - let results: SegmentQueryResultRow[] = []; - - const n = Math.min(datasourceList.length, maxResults); - for (let i = 0; i < n && results.length < maxResults; i++) { - const segments = ( - await Api.instance.get( - `/druid/coordinator/v1/datasources/${Api.encodePath(datasourceList[i])}?full`, - { cancelToken }, - ) - ).data?.segments; - if (!Array.isArray(segments)) continue; - - let segmentQueryResultRows: SegmentQueryResultRow[] = segments.map((segment: any) => { - const [start, end] = segment.interval.split('/'); - return { - segment_id: segment.identifier, - datasource: segment.dataSource, - start, - end, - interval: segment.interval, - version: segment.version, - shard_spec: deepGet(segment, 'shardSpec'), - partition_num: deepGet(segment, 'shardSpec.partitionNum') || 0, - size: segment.size, - num_rows: -1, - avg_row_size: -1, - num_replicas: -1, - replication_factor: -1, - is_available: -1, - is_active: -1, - is_realtime: -1, - is_published: -1, - is_overshadowed: -1, - }; - }); - - if (filtered.length) { - segmentQueryResultRows = segmentQueryResultRows.filter((d: SegmentQueryResultRow) => { - return filtered.every(filter => { - return booleanCustomTableFilter( - filter, - d[filter.id as keyof SegmentQueryResultRow], - ); - }); + let results = ( + await getApiArray( + `/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&includeRealtimeSegments${datasourceList + .map(d => `&datasources=${Api.encodePath(d)}`) + .join('')}`, + cancelToken, + ) + ).map((segment: any) => { + const [start, end] = segment.interval.split('/'); + return { + segment_id: segment.identifier, + datasource: segment.dataSource, + start, + end, + interval: segment.interval, + version: segment.version, + shard_spec: segment.shardSpec, + partition_num: segment.shardSpec?.partitionNum || 0, + size: segment.size, + num_rows: -1, + avg_row_size: -1, + num_replicas: -1, + replication_factor: segment.replicationFactor, + is_available: -1, + is_active: -1, + is_realtime: Number(segment.realtime), + is_published: -1, + is_overshadowed: Number(segment.overshadowed), + }; + }); + + if (filtered.length) { + results = results.filter((d: SegmentQueryResultRow) => { + return filtered.every(filter => { + return booleanCustomTableFilter( + filter, + d[filter.id as keyof SegmentQueryResultRow], + ); }); - } - - results = results.concat(segmentQueryResultRows); + }); } - return results.slice(page * pageSize, maxResults); + const maxResults = (page + 1) * pageSize; + return applySorting(results, sorted).slice(page * pageSize, maxResults); } else { throw new Error('must have SQL or coordinator access to load this view'); } @@ -585,7 +574,6 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment show: visibleColumns.shown('Segment ID'), accessor: 'segment_id', width: 280, - sortable: hasSql, filterable: allowGeneralFilter, Cell: row => ( <TableClickableCell @@ -618,7 +606,6 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment show: groupByInterval, accessor: 'interval', width: 120, - sortable: hasSql, defaultSortDesc: true, filterable: allowGeneralFilter, Cell: this.renderFilterableCell('interval'), @@ -629,7 +616,6 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment accessor: 'start', headerClassName: 'enable-comparisons', width: 180, - sortable: hasSql, defaultSortDesc: true, filterable: allowGeneralFilter, Cell: this.renderFilterableCell('start', true), @@ -640,7 +626,6 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment accessor: 'end', headerClassName: 'enable-comparisons', width: 180, - sortable: hasSql, defaultSortDesc: true, filterable: allowGeneralFilter, Cell: this.renderFilterableCell('end', true), @@ -650,7 +635,6 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment show: visibleColumns.shown('Version'), accessor: 'version', width: 180, - sortable: hasSql, defaultSortDesc: true, filterable: allowGeneralFilter, Cell: this.renderFilterableCell('version', true), @@ -796,7 +780,6 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment accessor: 'partition_num', width: 60, filterable: false, - sortable: hasSql, className: 'padded', }, { @@ -804,7 +787,6 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment show: visibleColumns.shown('Size'), accessor: 'size', filterable: false, - sortable: hasSql, defaultSortDesc: true, width: 120, className: 'padded', @@ -864,7 +846,7 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment }, { Header: twoLines('Replication factor', <i>(desired)</i>), - show: hasSql && visibleColumns.shown('Replication factor'), + show: visibleColumns.shown('Replication factor'), accessor: 'replication_factor', width: 80, filterable: false, @@ -891,7 +873,7 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment }, { Header: 'Is realtime', - show: hasSql && visibleColumns.shown('Is realtime'), + show: visibleColumns.shown('Is realtime'), id: 'is_realtime', accessor: row => String(Boolean(row.is_realtime)), Filter: BooleanFilterInput, @@ -909,7 +891,7 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment }, { Header: 'Is overshadowed', - show: hasSql && visibleColumns.shown('Is overshadowed'), + show: visibleColumns.shown('Is overshadowed'), id: 'is_overshadowed', accessor: row => String(Boolean(row.is_overshadowed)), Filter: BooleanFilterInput, diff --git a/web-console/src/views/services-view/services-view.tsx b/web-console/src/views/services-view/services-view.tsx index 88a5be35c52..000dadeb12c 100644 --- a/web-console/src/views/services-view/services-view.tsx +++ b/web-console/src/views/services-view/services-view.tsx @@ -48,6 +48,7 @@ import { formatBytes, formatBytesCompact, formatDurationWithMsIfNeeded, + getApiArray, hasPopoverOpen, LocalStorageBackedVisibility, LocalStorageKeys, @@ -257,24 +258,24 @@ ORDER BY if (capabilities.hasSql()) { services = await queryDruidSql({ query: ServicesView.SERVICE_SQL }, cancelToken); } else if (capabilities.hasCoordinatorAccess()) { - services = ( - await Api.instance.get('/druid/coordinator/v1/servers?simple', { cancelToken }) - ).data.map((s: any): ServiceResultRow => { - const hostParts = s.host.split(':'); - const port = parseInt(hostParts[1], 10); - return { - service: s.host, - service_type: s.type === 'indexer-executor' ? 'peon' : s.type, - tier: s.tier, - host: hostParts[0], - plaintext_port: port < 9000 ? port : -1, - tls_port: port < 9000 ? -1 : port, - curr_size: s.currSize, - max_size: s.maxSize, - start_time: '1970:01:01T00:00:00Z', - is_leader: 0, - }; - }); + services = (await getApiArray('/druid/coordinator/v1/servers?simple', cancelToken)).map( + (s: any): ServiceResultRow => { + const hostParts = s.host.split(':'); + const port = parseInt(hostParts[1], 10); + return { + service: s.host, + service_type: s.type === 'indexer-executor' ? 'peon' : s.type, + tier: s.tier, + host: hostParts[0], + plaintext_port: port < 9000 ? port : -1, + tls_port: port < 9000 ? -1 : port, + curr_size: s.currSize, + max_size: s.maxSize, + start_time: '1970:01:01T00:00:00Z', + is_leader: 0, + }; + }, + ); } else { throw new Error(`must have SQL or coordinator access`); } @@ -308,9 +309,10 @@ ORDER BY if (capabilities.hasOverlordAccess()) { auxiliaryQueries.push(async (services, cancelToken) => { try { - const workerInfos = ( - await Api.instance.get<WorkerInfo[]>('/druid/indexer/v1/workers', { cancelToken }) - ).data; + const workerInfos = await getApiArray<WorkerInfo>( + '/druid/indexer/v1/workers', + cancelToken, + ); const workerInfoLookup: Record<string, WorkerInfo> = lookupBy( workerInfos, diff --git a/web-console/src/views/supervisors-view/supervisors-view.tsx b/web-console/src/views/supervisors-view/supervisors-view.tsx index c9d75af9680..334339bc7cc 100644 --- a/web-console/src/views/supervisors-view/supervisors-view.tsx +++ b/web-console/src/views/supervisors-view/supervisors-view.tsx @@ -70,6 +70,7 @@ import { formatBytes, formatInteger, formatRate, + getApiArray, getDruidErrorMessage, hasPopoverOpen, isNumberLike, @@ -275,26 +276,22 @@ export class SupervisorsView extends React.PureComponent< return { ...supervisor, spec: JSONBig.parse(spec) }; }); } else if (capabilities.hasOverlordAccess()) { - const supervisorList = ( - await Api.instance.get('/druid/indexer/v1/supervisor?full', { cancelToken }) - ).data; - if (!Array.isArray(supervisorList)) { - throw new Error(`Unexpected result from /druid/indexer/v1/supervisor?full`); - } - supervisors = supervisorList.map((sup: any) => { - return { - supervisor_id: deepGet(sup, 'id'), - type: deepGet(sup, 'spec.tuningConfig.type'), - source: - deepGet(sup, 'spec.ioConfig.topic') || - deepGet(sup, 'spec.ioConfig.stream') || - 'n/a', - state: deepGet(sup, 'state'), - detailed_state: deepGet(sup, 'detailedState'), - spec: sup.spec, - suspended: Boolean(deepGet(sup, 'suspended')), - }; - }); + supervisors = (await getApiArray('/druid/indexer/v1/supervisor?full', cancelToken)).map( + (sup: any) => { + return { + supervisor_id: deepGet(sup, 'id'), + type: deepGet(sup, 'spec.tuningConfig.type'), + source: + deepGet(sup, 'spec.ioConfig.topic') || + deepGet(sup, 'spec.ioConfig.stream') || + 'n/a', + state: deepGet(sup, 'state'), + detailed_state: deepGet(sup, 'detailedState'), + spec: sup.spec, + suspended: Boolean(deepGet(sup, 'suspended')), + }; + }, + ); const firstSorted = sorted[0]; if (firstSorted) { diff --git a/web-console/src/views/tasks-view/tasks-view.tsx b/web-console/src/views/tasks-view/tasks-view.tsx index 2bad1a29f66..0e44c96413f 100644 --- a/web-console/src/views/tasks-view/tasks-view.tsx +++ b/web-console/src/views/tasks-view/tasks-view.tsx @@ -42,6 +42,7 @@ import { SMALL_TABLE_PAGE_SIZE, SMALL_TABLE_PAGE_SIZE_OPTIONS } from '../../reac import { Api, AppToaster } from '../../singletons'; import { formatDuration, + getApiArray, getDruidErrorMessage, hasPopoverOpen, LocalStorageBackedVisibility, @@ -174,8 +175,19 @@ ORDER BY cancelToken, ); } else if (capabilities.hasOverlordAccess()) { - const resp = await Api.instance.get(`/druid/indexer/v1/tasks`, { cancelToken }); - return TasksView.parseTasks(resp.data); + return (await getApiArray(`/druid/indexer/v1/tasks`, cancelToken)).map(d => { + return { + task_id: d.id, + group_id: d.groupId, + type: d.type, + created_time: d.createdTime, + datasource: d.dataSource, + duration: d.duration ? d.duration : 0, + error_msg: d.errorMsg, + location: d.location.host ? `${d.location.host}:${d.location.port}` : null, + status: d.statusCode === 'RUNNING' ? d.runnerStatusCode : d.statusCode, + }; + }); } else { throw new Error(`must have SQL or overlord access`); } @@ -188,22 +200,6 @@ ORDER BY }); } - static parseTasks = (data: any[]): TaskQueryResultRow[] => { - return data.map(d => { - return { - task_id: d.id, - group_id: d.groupId, - type: d.type, - created_time: d.createdTime, - datasource: d.dataSource, - duration: d.duration ? d.duration : 0, - error_msg: d.errorMsg, - location: d.location.host ? `${d.location.host}:${d.location.port}` : null, - status: d.statusCode === 'RUNNING' ? d.runnerStatusCode : d.statusCode, - }; - }); - }; - componentDidMount(): void { const { capabilities } = this.props; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
