This is an automated email from the ASF dual-hosted git repository. SvenO3 pushed a commit to branch 4633-move-dataset-count-display-into-dataset-overview in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit eabccf0cc3f3b2db11b44c8cb07630c95ffaaecd Author: Sven Oehler <[email protected]> AuthorDate: Wed Jun 24 17:16:00 2026 +0200 Add metrics to dataset overview --- .../daily-event-counts-chart.component.html | 29 +++ .../daily-event-counts-chart.component.scss} | 25 +- .../daily-event-counts-chart.component.ts | 92 +++++++ .../dataset-details-metrics.component.html | 74 ++++++ .../dataset-details-metrics.component.scss} | 19 +- .../dataset-details-metrics.component.ts | 279 +++++++++++++++++++++ .../dataset-details/dataset-details-tabs.ts | 5 + ui/src/app/dataset/dataset.routes.ts | 5 + 8 files changed, 495 insertions(+), 33 deletions(-) diff --git a/ui/src/app/dataset/components/dataset-details/dataset-details-metrics/daily-event-counts-chart/daily-event-counts-chart.component.html b/ui/src/app/dataset/components/dataset-details/dataset-details-metrics/daily-event-counts-chart/daily-event-counts-chart.component.html new file mode 100644 index 0000000000..2261573d00 --- /dev/null +++ b/ui/src/app/dataset/components/dataset-details/dataset-details-metrics/daily-event-counts-chart/daily-event-counts-chart.component.html @@ -0,0 +1,29 @@ +<!-- +~ Licensed to the Apache Software Foundation (ASF) under one or more +~ contributor license agreements. See the NOTICE file distributed with +~ this work for additional information regarding copyright ownership. +~ The ASF licenses this file to You under the Apache License, Version 2.0 +~ (the "License"); you may not use this file except in compliance with +~ the License. You may obtain a copy of the License at +~ +~ http://www.apache.org/licenses/LICENSE-2.0 +~ +~ Unless required by applicable law or agreed to in writing, software +~ distributed under the License is distributed on an "AS IS" BASIS, +~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~ See the License for the specific language governing permissions and +~ limitations under the License. +~ +--> + +<div fxLayout="column" fxLayoutGap="0.5rem"> + <h3 class="chart-title"> + {{ 'Events in the last 7 days (UTC)' | translate }} + </h3> + <div + echarts + class="daily-counts-chart" + data-cy="dataset-details-metrics-daily-counts-chart" + [options]="chartOptions" + ></div> +</div> diff --git a/ui/src/app/dataset/components/dataset-details/dataset-details-tabs.ts b/ui/src/app/dataset/components/dataset-details/dataset-details-metrics/daily-event-counts-chart/daily-event-counts-chart.component.scss similarity index 60% copy from ui/src/app/dataset/components/dataset-details/dataset-details-tabs.ts copy to ui/src/app/dataset/components/dataset-details/dataset-details-metrics/daily-event-counts-chart/daily-event-counts-chart.component.scss index db4637ac90..2b29309aaa 100644 --- a/ui/src/app/dataset/components/dataset-details/dataset-details-tabs.ts +++ b/ui/src/app/dataset/components/dataset-details/dataset-details-metrics/daily-event-counts-chart/daily-event-counts-chart.component.scss @@ -16,21 +16,14 @@ * */ -import { SpNavigationItem } from '@streampipes/shared-ui'; +.chart-title { + margin: 0; + font-size: 1.35rem; + font-weight: 600; + text-align: center; +} -export class SpDatasetDetailsTabs { - public getTabs(elementId: string): SpNavigationItem[] { - return [ - { - itemId: 'schema', - itemTitle: 'Event schema', - itemLink: ['datasets', elementId, 'schema'], - }, - { - itemId: 'events', - itemTitle: 'Latest events', - itemLink: ['datasets', elementId, 'events'], - }, - ]; - } +.daily-counts-chart { + height: 320px; + width: 100%; } diff --git a/ui/src/app/dataset/components/dataset-details/dataset-details-metrics/daily-event-counts-chart/daily-event-counts-chart.component.ts b/ui/src/app/dataset/components/dataset-details/dataset-details-metrics/daily-event-counts-chart/daily-event-counts-chart.component.ts new file mode 100644 index 0000000000..8f42281c7e --- /dev/null +++ b/ui/src/app/dataset/components/dataset-details/dataset-details-metrics/daily-event-counts-chart/daily-event-counts-chart.component.ts @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { Component, Input } from '@angular/core'; +import { EChartsOption } from 'echarts'; +import { NgxEchartsDirective } from 'ngx-echarts'; +import { TranslatePipe } from '@ngx-translate/core'; +import { + LayoutDirective, + LayoutGapDirective, +} from '@ngbracket/ngx-layout/flex'; + +export interface DailyEventCount { + label: string; + count: number; +} + +@Component({ + selector: 'sp-daily-event-counts-chart', + templateUrl: './daily-event-counts-chart.component.html', + styleUrls: ['./daily-event-counts-chart.component.scss'], + imports: [ + LayoutDirective, + LayoutGapDirective, + NgxEchartsDirective, + TranslatePipe, + ], +}) +export class DailyEventCountsChartComponent { + chartOptions: EChartsOption = this.makeChartOptions([]); + + @Input() + set dailyEventCounts(dailyEventCounts: DailyEventCount[]) { + this.chartOptions = this.makeChartOptions(dailyEventCounts ?? []); + } + + private makeChartOptions( + dailyEventCounts: DailyEventCount[], + ): EChartsOption { + return { + color: ['#0f7f8f'], + tooltip: { + trigger: 'axis', + valueFormatter: value => `${value} events`, + }, + grid: { + top: 24, + right: 24, + bottom: 32, + left: 56, + }, + xAxis: { + type: 'category', + data: dailyEventCounts.map(bucket => bucket.label), + axisTick: { + alignWithLabel: true, + }, + }, + yAxis: { + type: 'value', + name: 'Events', + minInterval: 1, + }, + series: [ + { + name: 'Events', + type: 'bar', + barMaxWidth: 100, + data: dailyEventCounts.map(bucket => bucket.count), + itemStyle: { + borderRadius: [4, 4, 0, 0], + }, + }, + ], + }; + } +} diff --git a/ui/src/app/dataset/components/dataset-details/dataset-details-metrics/dataset-details-metrics.component.html b/ui/src/app/dataset/components/dataset-details/dataset-details-metrics/dataset-details-metrics.component.html new file mode 100644 index 0000000000..d86080fef6 --- /dev/null +++ b/ui/src/app/dataset/components/dataset-details/dataset-details-metrics/dataset-details-metrics.component.html @@ -0,0 +1,74 @@ +<!-- +~ Licensed to the Apache Software Foundation (ASF) under one or more +~ contributor license agreements. See the NOTICE file distributed with +~ this work for additional information regarding copyright ownership. +~ The ASF licenses this file to You under the Apache License, Version 2.0 +~ (the "License"); you may not use this file except in compliance with +~ the License. You may obtain a copy of the License at +~ +~ http://www.apache.org/licenses/LICENSE-2.0 +~ +~ Unless required by applicable law or agreed to in writing, software +~ distributed under the License is distributed on an "AS IS" BASIS, +~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~ See the License for the specific language governing permissions and +~ limitations under the License. +~ +--> + +<sp-basic-nav-tabs + [spNavigationItems]="tabs" + [activeLink]="'metrics'" + [showBackLink]="true" + [backLinkTarget]="['datasets']" +> + <div nav fxLayout="row" fxLayoutAlign="end center"> + <button + mat-icon-button + color="accent" + class="mr-10" + [matTooltip]="'Refresh' | translate" + (click)="triggerUpdate()" + > + <i class="material-icons">refresh</i> + </button> + </div> + + @if (datasetNotFound) { + <div class="text-xl" fxFlex="100" fxLayoutAlign="center center"> + {{ 'The desired dataset was not found!' | translate }} + </div> + } @else if (dataset) { + <div + fxFlex="100" + fxLayout="column" + fxLayoutGap="1rem" + data-cy="dataset-details-metrics" + > + @if (loadingMetrics) { + <div + class="loading-state" + fxLayout="row" + fxLayoutAlign="center center" + > + <mat-spinner [diameter]="32" color="accent"> + {{ 'Loading' | translate }} + </mat-spinner> + </div> + } + + <sp-simple-metrics + [elementName]="dataset.measureName" + [lastPublishedLabel]="'Last event' | translate" + [statusValueLabel]="'Total events' | translate" + [lastTimestamp]="lastEventTimestamp" + [statusValue]="totalEventCount" + > + </sp-simple-metrics> + + <sp-daily-event-counts-chart + [dailyEventCounts]="dailyEventCounts" + ></sp-daily-event-counts-chart> + </div> + } +</sp-basic-nav-tabs> diff --git a/ui/src/app/dataset/components/dataset-details/dataset-details-tabs.ts b/ui/src/app/dataset/components/dataset-details/dataset-details-metrics/dataset-details-metrics.component.scss similarity index 60% copy from ui/src/app/dataset/components/dataset-details/dataset-details-tabs.ts copy to ui/src/app/dataset/components/dataset-details/dataset-details-metrics/dataset-details-metrics.component.scss index db4637ac90..cf54b55ee6 100644 --- a/ui/src/app/dataset/components/dataset-details/dataset-details-tabs.ts +++ b/ui/src/app/dataset/components/dataset-details/dataset-details-metrics/dataset-details-metrics.component.scss @@ -16,21 +16,6 @@ * */ -import { SpNavigationItem } from '@streampipes/shared-ui'; - -export class SpDatasetDetailsTabs { - public getTabs(elementId: string): SpNavigationItem[] { - return [ - { - itemId: 'schema', - itemTitle: 'Event schema', - itemLink: ['datasets', elementId, 'schema'], - }, - { - itemId: 'events', - itemTitle: 'Latest events', - itemLink: ['datasets', elementId, 'events'], - }, - ]; - } +.loading-state { + min-height: 3rem; } diff --git a/ui/src/app/dataset/components/dataset-details/dataset-details-metrics/dataset-details-metrics.component.ts b/ui/src/app/dataset/components/dataset-details/dataset-details-metrics/dataset-details-metrics.component.ts new file mode 100644 index 0000000000..b347cecb12 --- /dev/null +++ b/ui/src/app/dataset/components/dataset-details/dataset-details-metrics/dataset-details-metrics.component.ts @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { Component, OnInit } from '@angular/core'; +import { SpAbstractDatasetDetailsDirective } from '../abstract-dataset-details.directive'; +import { SpQueryResult } from '@streampipes/platform-services'; +import { SpBasicNavTabsComponent } from '@streampipes/shared-ui'; +import { + FlexDirective, + LayoutAlignDirective, + LayoutDirective, + LayoutGapDirective, +} from '@ngbracket/ngx-layout/flex'; +import { MatIconButton } from '@angular/material/button'; +import { MatTooltip } from '@angular/material/tooltip'; +import { MatProgressSpinner } from '@angular/material/progress-spinner'; +import { TranslatePipe } from '@ngx-translate/core'; +import { SpSimpleMetricsComponent } from '../../../../core-ui/monitoring/simple-metrics/simple-metrics.component'; +import { SpConfigurationRoutes } from '../../../../configuration/configuration.breadcrumb'; +import { catchError, finalize, forkJoin, map, Observable, of } from 'rxjs'; +import { + DailyEventCount, + DailyEventCountsChartComponent, +} from './daily-event-counts-chart/daily-event-counts-chart.component'; + +interface DayBucket extends DailyEventCount { + timestamp: number; + key: string; +} + +@Component({ + selector: 'sp-dataset-details-metrics', + templateUrl: './dataset-details-metrics.component.html', + styleUrls: ['./dataset-details-metrics.component.scss'], + imports: [ + SpBasicNavTabsComponent, + LayoutDirective, + LayoutAlignDirective, + LayoutGapDirective, + FlexDirective, + MatIconButton, + MatTooltip, + MatProgressSpinner, + TranslatePipe, + SpSimpleMetricsComponent, + DailyEventCountsChartComponent, + ], +}) +export class DatasetDetailsMetricsComponent + extends SpAbstractDatasetDetailsDirective + implements OnInit +{ + totalEventCount = 0; + lastEventTimestamp = 0; + loadingMetrics = false; + dailyEventCounts: DailyEventCount[] = []; + + ngOnInit(): void { + super.onInit(); + } + + onDatasetLoaded(): void { + this.breadcrumbService.updateBreadcrumb([ + SpConfigurationRoutes.BASE, + { label: 'Datasets', link: ['datasets'] }, + { label: this.dataset.measureName }, + { label: 'Metrics' }, + ]); + this.triggerUpdate(); + } + + triggerUpdate(): void { + if (!this.dataset) { + return; + } + + this.loadingMetrics = true; + const now = new Date(); + const dayBuckets = this.makeLastSevenDayBuckets(now); + + forkJoin({ + totalEventCount: this.loadTotalEventCount(), + latestEventTimestamp: this.loadLatestEventTimestamp(now), + dailyEventCounts: this.loadDailyEventCounts(dayBuckets, now), + }) + .pipe( + finalize(() => { + this.loadingMetrics = false; + }), + ) + .subscribe(result => { + this.totalEventCount = result.totalEventCount; + this.lastEventTimestamp = result.latestEventTimestamp; + this.dailyEventCounts = result.dailyEventCounts; + }); + } + + private loadTotalEventCount(): Observable<number> { + return this.datalakeRestService + .getMeasurementEntryCount(this.dataset.elementId) + .pipe(catchError(() => of(0))); + } + + private loadLatestEventTimestamp(now: Date): Observable<number> { + return this.datalakeRestService + .getData(this.dataset.measureName, { + endDate: now.getTime(), + startDate: 0, + limit: 1, + order: 'DESC', + missingValueBehaviour: 'empty', + columns: this.getRuntimeNames().toString(), + }) + .pipe( + map(result => this.extractLatestTimestamp(result)), + catchError(() => of(0)), + ); + } + + private loadDailyEventCounts( + dayBuckets: DayBucket[], + now: Date, + ): Observable<DayBucket[]> { + const firstRuntimeName = this.getRuntimeNames()[0]; + if (!firstRuntimeName) { + return of(dayBuckets); + } + + return this.datalakeRestService + .getData(this.dataset.measureName, { + endDate: now.getTime(), + startDate: dayBuckets[0].timestamp, + order: 'ASC', + missingValueBehaviour: 'empty', + columns: firstRuntimeName, + aggregationFunction: 'COUNT', + timeInterval: '1d', + fill: 0, + }) + .pipe( + map(result => this.normalizeDailyCounts(result, dayBuckets)), + catchError(() => of(dayBuckets)), + ); + } + + private extractLatestTimestamp(result: SpQueryResult): number { + const series = result?.allDataSeries?.find( + dataSeries => dataSeries.rows?.length > 0, + ); + const row = series?.rows?.[0]; + if (!row) { + return 0; + } + + const headers = result.headers?.length + ? result.headers + : series.headers; + const timestampIndex = this.getHeaderIndex(headers, 'time', 0); + return this.toTimestamp(row[timestampIndex]); + } + + private normalizeDailyCounts( + result: SpQueryResult, + dayBuckets: DayBucket[], + ): DayBucket[] { + const countsByDay = new Map( + dayBuckets.map(bucket => [bucket.key, bucket.count]), + ); + + result?.allDataSeries?.forEach(series => { + const headers = result.headers?.length + ? result.headers + : series.headers; + const timestampIndex = this.getHeaderIndex(headers, 'time', 0); + + series.rows?.forEach(row => { + const countIndex = this.getCountHeaderIndex(headers, row); + const key = this.toUtcDayKey(new Date(row[timestampIndex])); + if (countsByDay.has(key)) { + countsByDay.set( + key, + (countsByDay.get(key) ?? 0) + + this.toCount(row[countIndex]), + ); + } + }); + }); + + return dayBuckets.map(bucket => ({ + ...bucket, + count: countsByDay.get(bucket.key) ?? 0, + })); + } + + private makeLastSevenDayBuckets(now: Date): DayBucket[] { + const startTimestamp = Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate() - 6, + ); + + return Array.from({ length: 7 }, (_value, index) => { + const timestamp = startTimestamp + index * 24 * 60 * 60 * 1000; + const date = new Date(timestamp); + + return { + timestamp, + key: this.toUtcDayKey(date), + label: new Intl.DateTimeFormat(undefined, { + month: 'short', + day: 'numeric', + timeZone: 'UTC', + }).format(date), + count: 0, + }; + }); + } + + private getRuntimeNames(): string[] { + return (this.dataset.eventSchema?.eventProperties ?? []).map( + property => property.runtimeName, + ); + } + + private getHeaderIndex( + headers: string[] | undefined, + header: string, + fallback: number, + ): number { + const index = headers?.indexOf(header) ?? -1; + return index >= 0 ? index : fallback; + } + + private getCountHeaderIndex( + headers: string[] | undefined, + row: unknown[], + ): number { + const countIndex = + headers?.findIndex( + header => header === 'count' || header.startsWith('count_'), + ) ?? -1; + return countIndex >= 0 ? countIndex : Math.min(row.length - 1, 1); + } + + private toTimestamp(value: unknown): number { + const timestamp = + typeof value === 'number' + ? value + : new Date(String(value)).getTime(); + return Number.isNaN(timestamp) ? 0 : timestamp; + } + + private toCount(value: unknown): number { + const count = Number(value); + return Number.isFinite(count) ? count : 0; + } + + private toUtcDayKey(date: Date): string { + const year = date.getUTCFullYear(); + const month = String(date.getUTCMonth() + 1).padStart(2, '0'); + const day = String(date.getUTCDate()).padStart(2, '0'); + return `${year}-${month}-${day}`; + } +} diff --git a/ui/src/app/dataset/components/dataset-details/dataset-details-tabs.ts b/ui/src/app/dataset/components/dataset-details/dataset-details-tabs.ts index db4637ac90..92e29c31a8 100644 --- a/ui/src/app/dataset/components/dataset-details/dataset-details-tabs.ts +++ b/ui/src/app/dataset/components/dataset-details/dataset-details-tabs.ts @@ -26,6 +26,11 @@ export class SpDatasetDetailsTabs { itemTitle: 'Event schema', itemLink: ['datasets', elementId, 'schema'], }, + { + itemId: 'metrics', + itemTitle: 'Metrics', + itemLink: ['datasets', elementId, 'metrics'], + }, { itemId: 'events', itemTitle: 'Latest events', diff --git a/ui/src/app/dataset/dataset.routes.ts b/ui/src/app/dataset/dataset.routes.ts index b1d8417bb0..932cf1e26d 100644 --- a/ui/src/app/dataset/dataset.routes.ts +++ b/ui/src/app/dataset/dataset.routes.ts @@ -20,6 +20,7 @@ import { Routes } from '@angular/router'; import { DatalakeConfigurationComponent } from './components/datalake-configuration/datalake-configuration.component'; import { DatasetDetailsSchemaComponent } from './components/dataset-details/dataset-details-schema/dataset-details-schema.component'; import { DatasetDetailsEventsComponent } from './components/dataset-details/dataset-details-events/dataset-details-events.component'; +import { DatasetDetailsMetricsComponent } from './components/dataset-details/dataset-details-metrics/dataset-details-metrics.component'; export const DATASET_ROUTES: Routes = [ { @@ -42,6 +43,10 @@ export const DATASET_ROUTES: Routes = [ path: 'schema', component: DatasetDetailsSchemaComponent, }, + { + path: 'metrics', + component: DatasetDetailsMetricsComponent, + }, { path: 'events', component: DatasetDetailsEventsComponent,
