rusackas commented on code in PR #41184: URL: https://github.com/apache/superset/pull/41184#discussion_r3565628555
########## superset/common/grouping_sets.py: ########## @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +SQL building blocks for the pivot-table non-additive totals optimization +(SIP.md, phase 3b). When a datasource engine reports +``supports_grouping_sets``, the N per-rollup-level queries can be collapsed into +a single ``GROUPING SETS`` query: the database computes every level in one scan, +and each returned row is attributed to its level via ``GROUPING()`` markers. + +These are the engine-agnostic SQL primitives. Wiring them into the query context +(emitting one query and splitting the result back into per-level results) is the +remaining integration; see SIP.md. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Final + +import pandas as pd +from sqlalchemy import func, tuple_ +from sqlalchemy.sql.elements import ColumnElement + +# Suffix for the per-column GROUPING() marker columns added to a GROUPING SETS +# query. Chosen to be unlikely to collide with a real metric/column label. +GROUPING_MARKER_SUFFIX: Final = "__superset_grouping" + + +def grouping_marker_label(column_label: str) -> str: + """The output label of the GROUPING() marker for a groupby column.""" + return f"{column_label}{GROUPING_MARKER_SUFFIX}" + + +def grouping_sets_clause( + groups: Sequence[Sequence[ColumnElement]], +) -> ColumnElement: + """ + Build a ``GROUP BY GROUPING SETS (...)`` clause from rollup column groups. + + Each group is the set of columns grouped at one rollup level; the empty + group ``()`` is the grand total. For example, groups ``[[a, b], [a], []]`` + produce ``GROUPING SETS ((a, b), (a), ())``. + + :param groups: one column list per rollup level + :return: a clause element suitable for ``select(...).group_by(...)`` + """ + return func.grouping_sets(*[tuple_(*group) for group in groups]) + + +def grouping_id_column(column: ColumnElement, label: str) -> ColumnElement: + """ + Build a ``GROUPING(col) AS label`` marker column. + + In a ``GROUPING SETS`` result, ``GROUPING(col)`` is ``0`` when ``col`` is + part of the row's grouping level and ``1`` when it has been rolled up + (aggregated away). Selecting one marker per groupby column lets the caller + attribute each returned row to its rollup level when splitting the single + result back into per-level results. + + :param column: the groupby column to probe + :param label: the output label for the marker (see ``grouping_marker_label``) + :return: the labelled ``GROUPING(col)`` column + """ + return func.grouping(column).label(label) + + +def split_grouping_sets_result( + df: pd.DataFrame, + levels: Sequence[Sequence[str]], + groupby_columns: Sequence[str], +) -> list[pd.DataFrame]: + """ + Split a combined ``GROUPING SETS`` result into one DataFrame per rollup + level, the inverse of {@link grouping_sets_clause}. Review Comment: Fixed, swapped it for a proper Sphinx `:func:` reference. ########## superset-frontend/plugins/plugin-chart-pivot-table/src/PivotTableChart.tsx: ########## @@ -383,22 +338,45 @@ export default function PivotTableChart(props: PivotTableProps) { const unpivotedData = useMemo( () => - data.reduce( - (acc: DataRecord[], record: DataRecord) => [ - ...acc, - ...metricNames + // `data` is now one entry per rollup level. Tag every record with the + // row/column dimension labels of the level that produced it (mirroring + // the METRIC_KEY injection used for the full rows/cols below) so + // PivotData can slot each pre-computed value without re-aggregating. + // buildGroupbyCombinations already applied transposePivot, so the level's + // groupby is display-oriented and is not transposed again here. + data.flatMap((query: QueryData) => { + let levelRows = query.groupby.rows.map(getColumnLabel); + let levelCols = query.groupby.columns.map(getColumnLabel); + if (metricsLayout === MetricsLayoutEnum.ROWS) { + levelRows = combineMetric + ? [...levelRows, METRIC_KEY] + : [METRIC_KEY, ...levelRows]; + } else { + levelCols = combineMetric + ? [...levelCols, METRIC_KEY] + : [METRIC_KEY, ...levelCols]; + } + return query.data.flatMap((record: DataRecord) => + metricNames .map((name: string) => ({ ...record, [METRIC_KEY]: name, value: record[name], // Mark currency column for per-cell currency detection in aggregators __currencyColumn: currencyCodeColumn, + // The level this record belongs to (used by PivotData placement). + // Namespaced with a `__` prefix (like `__metricKey` below) so it + // can't collide with a real dataset column named `rows`/`columns`. + __rows: levelRows, + __columns: levelCols, + // Identify the metric pseudo-dimension so PivotData can feed the + // metric-collapsed totals (the opposite "Total" axis + corner). + __metricKey: METRIC_KEY, })) - .filter(record => record.value !== null), - ], - [], - ), - [data, metricNames, currencyCodeColumn], + .filter(r => r.value !== null), Review Comment: Already fixed in the latest push, same finding as the thread above. ########## superset-frontend/plugins/plugin-chart-pivot-table/src/plugin/transformProps.ts: ########## @@ -88,20 +98,63 @@ export default function transformProps(chartProps: ChartProps<QueryFormData>) { emitCrossFilters, theme, } = chartProps; - const { - data, - colnames, - coltypes, - detected_currency: detectedCurrency, - } = queriesData[0]; + const groupbyCombinations = buildGroupbyCombinations( + formData as PivotTableQueryFormData, + ); + const metricsArr = ensureIsArray(formData.metrics); + let data: QueryData[]; + if (allMetricsAdditive(metricsArr)) { + // Additive fast-path: a single full-detail query was issued; synthesize + // each rollup level by reducing the leaf rows on the client (see SIP.md). + const leafRows = queriesData[0].data; + const metricReducers: Record<string, RollupReducer> = {}; + metricsArr.forEach(metric => { + metricReducers[getMetricLabel(metric)] = additiveReducerFor(metric); + }); + const labelLevels = groupbyCombinations.map(combination => ({ + rows: combination.rows.map(getColumnLabel), + columns: combination.columns.map(getColumnLabel), + })); + const synthesized = synthesizeAdditiveLevels( + leafRows, + labelLevels, + metricReducers, + ); Review Comment: Good catch, `MIN`/`MAX` were getting force-coerced through `Number()` which drops temporal/string values to null. Fixed by comparing raw values for min/max and only coercing for `sum`. ########## superset-frontend/plugins/plugin-chart-pivot-table/src/plugin/buildQuery.ts: ########## @@ -54,6 +59,39 @@ export default function buildQuery(formData: PivotTableQueryFormData) { } return col; }); +} + +export default function buildQuery(formData: PivotTableQueryFormData) { + const { extra_form_data } = formData; + const time_grain_sqla = + extra_form_data?.time_grain_sqla || formData.time_grain_sqla; Review Comment: That's not right, already populates from by default. The here is only used to tag the BASE_AXIS column, and there is already a passing test () covering the override. ########## superset-frontend/plugins/plugin-chart-pivot-table/test/react-pivottable/tableRenders.test.tsx: ########## @@ -149,18 +202,83 @@ test('TableRenderer renders col totals row when colTotals is enabled', () => { test('TableRenderer renders grand total when both totals are enabled', () => { const props = buildDefaultProps({ + data: TAGGED_COUNT_DATA, + vals: ['value'], tableOptions: { rowTotals: true, colTotals: true }, }); renderWithTheme(<TableRenderer {...props} />); - // The grand total cell should show "4" (total record count). + // The grand total cell shows the DB-computed "4" (total record count). const grandTotalCells = screen .getAllByRole('gridcell') .filter(cell => cell.classList.contains('pvtGrandTotal')); expect(grandTotalCells.length).toBe(1); expect(grandTotalCells[0]).toHaveTextContent('4'); Review Comment: does a substring match by default, so satisfies . Test passes as written. ########## superset-frontend/plugins/plugin-chart-pivot-table/src/react-pivottable/utilities.ts: ########## @@ -1088,76 +1117,108 @@ class PivotData { } processRecord(record: PivotRecord): void { - // this code is called in a tight loop + // this code is called in a tight loop. + // Each record is tagged (in PivotTableChart) with `__rows`/`__columns`: + // the dimension labels of the rollup level that produced it. The database + // has already aggregated that level, so we place the value into exactly + // one slot and store it verbatim (no re-aggregation). A key shorter than + // the full dimension list identifies a subtotal level. Records without + // tags (e.g. direct unit-test construction) fall back to the full + // dimension lists, i.e. they are treated as leaf rows. The `__` prefix + // keeps these rollup tags from colliding with a real dataset column + // named `rows`/`columns`. + const recordRows = (record.__rows as unknown as string[]) ?? null; + const recordColumns = (record.__columns as unknown as string[]) ?? null; + const levelRows = recordRows ?? (this.props.rows as string[]); + const levelColumns = recordColumns ?? (this.props.cols as string[]); + const colKey: string[] = []; const rowKey: string[] = []; - (this.props.cols as string[]).forEach((col: string) => { + levelColumns.forEach((col: string) => { colKey.push(col in record ? String(record[col]) : 'null'); }); - (this.props.rows as string[]).forEach((row: string) => { + levelRows.forEach((row: string) => { rowKey.push(row in record ? String(record[row]) : 'null'); }); - this.allTotal.push(record); - - const rowStart = this.subtotals.rowEnabled ? 1 : Math.max(1, rowKey.length); - const colStart = this.subtotals.colEnabled ? 1 : Math.max(1, colKey.length); - - let isRowSubtotal; - let isColSubtotal; - for (let ri = rowStart; ri <= rowKey.length; ri += 1) { - isRowSubtotal = ri < rowKey.length; - const fRowKey = rowKey.slice(0, ri); - const flatRowKey = flatKey(fRowKey); - if (!this.rowTotals[flatRowKey]) { - this.rowKeys.push(fRowKey); - this.rowTotals[flatRowKey] = this.getFormattedAggregator( - record, - rowKey, - )(this, fRowKey, []); + const flatRowKey = flatKey(rowKey); + const flatColKey = flatKey(colKey); + + const isColSubtotal = colKey.length < (this.props.cols as string[]).length; + const isRowSubtotal = rowKey.length < (this.props.rows as string[]).length; + + // Register/create the column-total slot the first time this colKey is seen. + if (colKey.length > 0 && !this.colTotals[flatColKey]) { + if (!isColSubtotal || this.subtotals.colEnabled) { + this.colKeys.push(colKey); } - this.rowTotals[flatRowKey].push(record); - this.rowTotals[flatRowKey].isSubtotal = isRowSubtotal; + this.colTotals[flatColKey] = this.getFormattedAggregator(record, colKey)( + this, + [], + colKey, + ); } - - for (let ci = colStart; ci <= colKey.length; ci += 1) { - isColSubtotal = ci < colKey.length; - const fColKey = colKey.slice(0, ci); - const flatColKey = flatKey(fColKey); - if (!this.colTotals[flatColKey]) { - this.colKeys.push(fColKey); - this.colTotals[flatColKey] = this.getFormattedAggregator( - record, + // Register/create the row-total slot the first time this rowKey is seen. + if (rowKey.length > 0 && !this.rowTotals[flatRowKey]) { + if (!isRowSubtotal || this.subtotals.rowEnabled) { + this.rowKeys.push(rowKey); + } + this.rowTotals[flatRowKey] = this.getFormattedAggregator(record, rowKey)( + this, + rowKey, + [], + ); + } + // Create the body-cell slot. + if (rowKey.length > 0 && colKey.length > 0) { + if (!this.tree[flatRowKey]) { + this.tree[flatRowKey] = {}; + } + if (!this.tree[flatRowKey][flatColKey]) { + this.tree[flatRowKey][flatColKey] = this.getFormattedAggregator(record)( + this, + rowKey, colKey, - )(this, [], fColKey); + ); } + } + + // Place the value in exactly one slot, determined by the level. + if (rowKey.length === 0 && colKey.length === 0) { + this.allTotal.push(record); + } else if (rowKey.length === 0) { this.colTotals[flatColKey].push(record); this.colTotals[flatColKey].isSubtotal = isColSubtotal; + } else if (colKey.length === 0) { + this.rowTotals[flatRowKey].push(record); + this.rowTotals[flatRowKey].isSubtotal = isRowSubtotal; + } else { + this.tree[flatRowKey][flatColKey].push(record); + this.tree[flatRowKey][flatColKey].isRowSubtotal = isRowSubtotal; + this.tree[flatRowKey][flatColKey].isColSubtotal = isColSubtotal; + this.tree[flatRowKey][flatColKey].isSubtotal = + isRowSubtotal || isColSubtotal; } - // And now fill in for all the sub-cells. - for (let ri = rowStart; ri <= rowKey.length; ri += 1) { - isRowSubtotal = ri < rowKey.length; - const fRowKey = rowKey.slice(0, ri); - const flatRowKey = flatKey(fRowKey); - if (!this.tree[flatRowKey]) { - this.tree[flatRowKey] = {}; + // Metric-collapse totals. The metric is a pseudo-dimension always present on + // one axis, so no rollup level produces an empty key on that axis -- which + // would leave the opposite "Total" axis and the grand-total corner empty. + // When a record's axis holds only the metric (no real dims there), its value + // is also the collapsed total for that axis, so mirror it into rowTotals / + // colTotals / allTotal. (For a single metric this equals the metric column; + // for multiple metrics it is the last metric -- a cross-metric total is not + // well defined and is left as future work.) + const metricKey = record.__metricKey as unknown as string | undefined; + if (metricKey) { + const realColCount = levelColumns.filter(c => c !== metricKey).length; + const realRowCount = levelRows.filter(r => r !== metricKey).length; + if (levelColumns.includes(metricKey) && realColCount === 0) { + if (rowKey.length === 0) this.allTotal.push(record); + else this.rowTotals[flatRowKey]?.push(record); } - for (let ci = colStart; ci <= colKey.length; ci += 1) { - isColSubtotal = ci < colKey.length; - const fColKey = colKey.slice(0, ci); - const flatColKey = flatKey(fColKey); - if (!this.tree[flatRowKey][flatColKey]) { - this.tree[flatRowKey][flatColKey] = this.getFormattedAggregator( - record, - )(this, fRowKey, fColKey); - } - this.tree[flatRowKey][flatColKey].push(record); - - this.tree[flatRowKey][flatColKey].isRowSubtotal = isRowSubtotal; - this.tree[flatRowKey][flatColKey].isColSubtotal = isColSubtotal; - this.tree[flatRowKey][flatColKey].isSubtotal = - isRowSubtotal || isColSubtotal; + if (levelRows.includes(metricKey) && realRowCount === 0) { + if (colKey.length === 0) this.allTotal.push(record); + else this.colTotals[flatColKey]?.push(record); } Review Comment: Different toggle: / gate intermediate subtotal levels, not the fully-collapsed totals level this mirrors into. When / are off, never requests that level's query in the first place, so never sees these records to begin with. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
