EnxDev commented on code in PR #43718:
URL: https://github.com/apache/superset/pull/43718#discussion_r3902936561
##########
superset/charts/client_processing.py:
##########
@@ -216,14 +646,58 @@ def pivot_df( # pylint: disable=too-many-locals,
too-many-arguments, too-many-s
subtotal_values = subtotal_values.apply(
pd.to_numeric, errors="coerce"
)
- subtotal = pivot_v2_aggfunc_map[aggfunc](subtotal_values,
axis=0)
+ if percent_mode:
+ subtotal = subtotal_values.apply(
+ lambda series: _reduce(series,
collapse(series.to_frame())[1])
+ )
+ else:
+ subtotal = pivot_v2_aggfunc_map[aggfunc](subtotal_values,
axis=0)
depth = groups.nlevels - len(subgroup) - 1
total = metric_name if level == 0 else __("Subtotal")
subtotal.name = tuple([*subgroup, total, *([""] * depth)]) #
noqa: C409
# insert row after subgroup
df = pd.concat(
[df[: slice_.stop], subtotal.to_frame().T, df[slice_.stop
:]]
)
+ inserted_rows.append(subtotal.name)
+ row_prefix_depth[subtotal.name] = level
+
+ if percent_mode and not apply_metrics_on_rows and rollup_levels:
Review Comment:
Could we keep the database rollups enabled when metrics are on rows? Both
rollup paths are skipped for this layout, so non-additive metrics fall back to
leaf-derived totals. For example, AVG leaves of 10/20 with a DB column rollup
of 18 export as 33.3%/66.7% instead of the chart's 55.6%/111.1%. A GROUPING
SETS test with `metricsLayout: ROWS` would cover this.
##########
superset/charts/client_processing.py:
##########
@@ -75,6 +83,368 @@ def get_column_key(label: tuple[str, ...], metrics:
list[str]) -> tuple[Any, ...
return tuple(parts)
+# How a metric's rollup total is derived from its cells, mirroring
+# `additiveReducerFor` in the pivot plugin's `plugin/utilities.ts`: SUM and
+# COUNT add up, MIN takes the lowest, MAX the highest. Everything else (saved
+# metrics, adhoc SQL, AVG, ...) is non-additive and has no correct answer at
+# this layer, so it falls back to summing the cells.
+_ROLLUP_REDUCERS: dict[str, str] = {"MIN": "min", "MAX": "max"}
+DEFAULT_ROLLUP_REDUCER = "sum"
+
+
+def split_grouping_sets_levels(
+ df: pd.DataFrame,
+) -> tuple[pd.DataFrame, dict[frozenset[str], pd.DataFrame]]:
+ """
+ Separate a GROUPING SETS result into its leaf frame and rollup levels.
+
+ A pivot chart with non-additive metrics asks for every rollup level in one
+ frame, tagging each row with a ``GROUPING()`` marker per groupby column
+ (see ``common/grouping_sets.py``): ``0`` where the column is grouped at
that
+ row's level, ``1`` where it has been rolled up. The rollup rows must not be
+ pivoted as ordinary rows -- their collapsed dimensions are NULL, so they
+ would add phantom rows and columns and inflate every denominator.
+
+ They are not discardable either: for a non-additive metric the database
+ rollup is the only correct total, and re-deriving one from the leaf cells
+ gives a different number (the mean of means, say, rather than the mean).
+ The chart divides by these values, so the export has to as well.
+
+ :return: the leaf frame, and each rollup level keyed by its grouped columns
+ """
+ markers = [
+ column
+ for column in df.columns
+ if isinstance(column, str) and column.endswith(GROUPING_MARKER_SUFFIX)
+ ]
+ if not markers:
+ return df, {}
+
+ grouped_of = {marker: marker[: -len(GROUPING_MARKER_SUFFIX)] for marker in
markers}
+ levels: dict[frozenset[str], pd.DataFrame] = {}
+ leaf = df
+ for keys, rows_at_level in df.groupby(markers, sort=False):
+ # `groupby` yields a scalar key for a single column and a tuple beyond.
+ marker_values = keys if isinstance(keys, tuple) else (keys,)
+ grouped = frozenset(
+ grouped_of[marker]
+ for marker, rolled_up in zip(markers, marker_values, strict=True)
+ if not rolled_up
+ )
+ level = rows_at_level.drop(columns=markers).reset_index(drop=True)
+ levels[grouped] = level
+ if len(grouped) == len(markers):
+ leaf = level
+ return leaf, levels
+
+
+def get_metric_rollup_reducers(
+ metrics: list[Any], verbose_map: Optional[dict[str, Any]] = None
+) -> dict[str, str]:
+ """Map each metric's label to the reducer that rolls its cells up."""
+ reducers: dict[str, str] = {}
+ for metric in metrics:
+ reducer = DEFAULT_ROLLUP_REDUCER
+ if isinstance(metric, dict) and metric.get("expressionType") ==
"SIMPLE":
+ reducer = _ROLLUP_REDUCERS.get(
+ metric.get("aggregate") or "", DEFAULT_ROLLUP_REDUCER
+ )
+ reducers[get_metric_name(metric, verbose_map)] = reducer
+ return reducers
+
+
+def _collapsed_metric(present: list[Any], metrics: list[str]) -> Any:
+ """
+ The metric a rollup spanning `present` stands for.
+
+ A total that collapses the metric axis is undefined in the renderer, which
+ resolves it to the last metric pushed into the shared slot (see the
+ ``metricAxis`` handling in ``react-pivottable/utilities.ts``). Mirror that
+ by taking the last metric in the configured order, so exported percentages
+ match the chart rather than summing metrics that share no unit.
+ """
+ distinct = set(present)
+ if len(distinct) == 1:
+ return distinct.pop()
+ for metric in reversed(metrics):
+ if metric in distinct:
+ return metric
+ return None
+
+
+def _broadcast(total: pd.Series, block: pd.DataFrame, axis: int) ->
pd.DataFrame:
+ """Spread a per-row (`axis` 0) or per-column (`axis` 1) total over
`block`."""
+ if axis == 0:
+ spread = pd.concat([total] * len(block.columns), axis=1)
+ spread.columns = block.columns
+ return spread
+ return pd.DataFrame(
+ np.tile(total.reindex(block.columns).to_numpy(), (len(block.index),
1)),
+ index=block.index,
+ columns=block.columns,
+ )
+
+
+def _metric_of_column(column: Any, metric_level: int) -> Any:
+ """The metric a pivoted column belongs to."""
+ return column[metric_level] if isinstance(column, tuple) else column
+
+
+def _reduce(
+ data: Union[pd.DataFrame, pd.Series],
+ reducer: str,
+ axis: Optional[int] = None,
+) -> Any:
+ """Apply a rollup reducer (``sum``/``min``/``max``), skipping empty
cells."""
+ method = getattr(data, reducer)
+ return method(axis=axis) if axis is not None else method()
+
+
+def _keyed_rollup(
+ frame: pd.DataFrame, dimensions: list[str]
+) -> dict[tuple[str, ...], dict[str, Any]]:
+ """Index a rollup level by its grouped dimension values, as strings."""
+ filled = frame.fillna("SUPERSET_PANDAS_NAN")
+ return {
+ tuple(str(record[dimension]) for dimension in dimensions): record
+ for record in filled.to_dict("records")
+ }
+
+
+def _rollup_key(
+ label: Any, depth: int, metric_level: int, is_column: bool
+) -> tuple[str, ...]:
+ """The grouped dimension values a pivoted row or column label carries."""
+ parts = list(label) if isinstance(label, tuple) else [label]
+ if is_column:
+ # The column label interleaves the metric with the dimension values.
+ parts = [part for index, part in enumerate(parts) if index !=
metric_level]
+ return tuple(str(part) for part in parts[:depth])
+
+
+def _apply_rollup_totals( # pylint: disable=too-many-arguments,too-many-locals
+ df: pd.DataFrame,
+ rows: list[str],
+ columns: list[str],
+ metrics: list[str],
+ rollup_levels: dict[frozenset[str], pd.DataFrame],
+ metric_level: int,
+ row_prefix_depth: dict[Any, int],
+ column_prefix_depth: dict[Any, int],
+) -> pd.DataFrame:
+ """
+ Replace inserted totals with the values the database computed.
+
+ A total grouping ``i`` row dimensions and ``j`` column dimensions is
exactly
+ the rollup level over ``rows[:i] + columns[:j]``, which
+ ``buildGroupbyCombinations`` requests whenever the chart displays that
+ total. Reading it keeps the export equal to the chart for a non-additive
+ metric, where reducing the leaf cells gives a different number.
+
+ Any total whose level or key is absent keeps its leaf-derived value, so a
+ missing level degrades to the previous behaviour rather than to a blank.
+ """
+ metric_names = set(metrics)
+
+ def metric_of(column: Any) -> Any:
+ name = _metric_of_column(column, metric_level)
+ return (
+ name if name in metric_names else _collapsed_metric(list(metrics),
metrics)
+ )
+
+ def lookup(row: Any, column: Any) -> Any:
+ row_depth = row_prefix_depth.get(row, len(rows))
+ column_depth = column_prefix_depth.get(column, len(columns))
+ level = rollup_levels.get(frozenset(rows[:row_depth] +
columns[:column_depth]))
+ if level is None:
+ return None
+ key = _rollup_key(row, row_depth, metric_level, is_column=False) +
_rollup_key(
+ column, column_depth, metric_level, is_column=True
+ )
+ record = _keyed_rollup(level, rows[:row_depth] +
columns[:column_depth]).get(
+ key
+ )
+ return record.get(metric_of(column)) if record else None
+
+ # Index positionally: a tuple label on a MultiIndex is ambiguous to `.loc`,
+ # which resolves some of these keys to the wrong row.
+ for column_position, column in enumerate(df.columns):
+ for row_position, row in enumerate(df.index):
+ if row not in row_prefix_depth and column not in
column_prefix_depth:
+ continue # a leaf cell, already carrying its own value
+ value = lookup(row, column)
+ if value is not None and not pd.isna(value):
+ df.iloc[row_position, column_position] = value
+ return df
+
+
+def _rollup_denominators( # pylint: disable=too-many-arguments,too-many-locals
+ df: pd.DataFrame,
+ mode: str,
+ rows: list[str],
+ columns: list[str],
+ metrics: list[str],
+ rollup_levels: dict[frozenset[str], pd.DataFrame],
+ metric_level: int,
+ row_prefix_depth: dict[Any, int],
+ column_prefix_depth: dict[Any, int],
+) -> Optional[pd.DataFrame]:
+ """
+ Each cell's denominator, taken from the database-computed rollup levels.
+
+ A percent mode makes the chart request the level its denominator needs: for
+ "% of row" the cell's own row with the columns collapsed, for "% of column"
+ the reverse, for "% of total" both (see ``buildGroupbyCombinations``). A
+ subtotal divides by its own prefix, not by the grand total -- an "EU"
+ subtotal row divides by the ``{region}`` rollup.
+
+ For a non-additive metric these are the only correct totals. Cells whose
+ level or key is absent come back NaN, leaving the caller to fall back to a
+ denominator derived from the leaf cells.
+ """
+ if not rollup_levels:
+ return None
+ metric_names = set(metrics)
+
+ def metric_of(column: Any) -> Any:
+ name = _metric_of_column(column, metric_level)
+ return (
+ name if name in metric_names else _collapsed_metric(list(metrics),
metrics)
+ )
+
+ def denominator(row: Any, column: Any) -> Any:
+ row_depth = row_prefix_depth.get(row, len(rows))
+ column_depth = column_prefix_depth.get(column, len(columns))
+ if mode == ShowValuesAs.PERCENT_OF_ROW:
+ grouped, key = (
+ rows[:row_depth],
+ _rollup_key(row, row_depth, metric_level, is_column=False),
+ )
+ elif mode == ShowValuesAs.PERCENT_OF_COLUMN:
+ grouped, key = (
+ columns[:column_depth],
+ _rollup_key(column, column_depth, metric_level,
is_column=True),
+ )
+ else:
+ grouped, key = [], ()
+ level = rollup_levels.get(frozenset(grouped))
+ if level is None:
+ return None
+ record = _keyed_rollup(level, grouped).get(key)
+ return record.get(metric_of(column)) if record else None
+
+ built = pd.DataFrame(
+ [[denominator(row, column) for column in df.columns] for row in
df.index],
+ index=df.index,
+ columns=df.columns,
+ )
+ return built.apply(pd.to_numeric, errors="coerce").astype(float)
+
+
+def _apply_show_values_as( # pylint: disable=too-many-arguments
+ df: pd.DataFrame,
+ mode: str,
+ axis: dict[str, int],
+ metrics: list[str],
+ combine_metrics: bool,
+ inserted_rows: list[Any],
+ inserted_columns: list[Any],
+ reducers: dict[str, str],
+ denominators: Optional[pd.DataFrame] = None,
+) -> pd.DataFrame:
+ """
+ Express each cell as a fraction of its row, column, or grand total.
+
+ Mirrors the client's ``fractionOf`` aggregator in
+ ``plugin-chart-pivot-table/src/react-pivottable/utilities.ts``. Two details
+ it inherits from there:
+
+ - Denominators are summed over leaf cells only. Totals and subtotals
+ inserted into the frame are numerators like any other cell -- a "% of
+ row" grand total row reads ``column total / grand total``, not the sum of
+ the fractions above it.
+ - A total is rolled up within a single metric, so a cell is never divided
by
+ a total that mixes in another metric, and each metric uses its own
+ reducer -- a MIN/MAX metric divides by the row's minimum/maximum rather
+ than its sum. A total that collapses the metric axis resolves to a single
+ metric the way the renderer does; see ``_collapsed_metric``.
+
+ A zero denominator yields NaN (blank) rather than infinity, matching
+ ``pandas_postprocessing.pivot``'s ``show_values_as``.
+ """
+ numeric = df.apply(pd.to_numeric, errors="coerce").astype(float)
+ is_multi_index = isinstance(df.columns, pd.MultiIndex)
+ # `combine_metrics` has already moved the metric to the lowest column
level.
+ metric_level = df.columns.nlevels - 1 if combine_metrics and
is_multi_index else 0
+ metric_names = set(metrics)
+ metric_of_column = [
+ key if key in metric_names else None
+ for key in df.columns.get_level_values(metric_level)
+ ]
+ leaf_rows = ~df.index.isin(inserted_rows)
+ leaf_columns = ~df.columns.isin(inserted_columns)
+
+ derived = pd.DataFrame(np.nan, index=numeric.index,
columns=numeric.columns)
+ for metric in dict.fromkeys(metric_of_column):
+ selection = np.array([column == metric for column in metric_of_column])
+ denominator_selection = (
+ selection if metric is not None else np.ones(len(selection),
dtype=bool)
+ )
+ # Derive the reducer from the columns forming the denominator, not from
+ # the numerator's own label: a total column carries a total label, but
+ # must still divide by a rollup of the metric it totals.
+ denominator_metric = _collapsed_metric(
+ [
+ column_metric
+ for column_metric, keep in zip(
+ metric_of_column, denominator_selection, strict=True
+ )
+ if keep and column_metric is not None
+ ],
+ metrics,
+ )
+ reducer = reducers.get(str(denominator_metric), DEFAULT_ROLLUP_REDUCER)
+ if denominator_metric is not None:
+ denominator_selection = denominator_selection & np.array(
+ [column == denominator_metric for column in metric_of_column]
+ )
+ block = numeric.loc[:, selection]
+ if mode == ShowValuesAs.PERCENT_OF_TOTAL:
+ leaf = numeric.loc[leaf_rows, leaf_columns & denominator_selection]
+ # Reduce through pandas, not numpy: a sparse pivot leaves NaN in
+ # cells whose group had no rows, and numpy would propagate that to
+ # the grand total, blanking every cell.
+ grand_total = _reduce(_reduce(leaf, reducer, axis=0), reducer)
+ group_denominator = pd.DataFrame(
+ np.nan if pd.isna(grand_total) else grand_total,
+ index=block.index,
+ columns=block.columns,
+ )
+ else:
+ summed, divided = (
+ (axis["rows"], axis["columns"])
+ if mode == ShowValuesAs.PERCENT_OF_COLUMN
+ else (axis["columns"], axis["rows"])
+ )
+ # The metric lives on the column axis, so only a rollup taken along
+ # that axis has to stay within one metric.
+ leaf = (
+ numeric.loc[:, leaf_columns & denominator_selection]
+ if summed == 1
+ else numeric.loc[leaf_rows, :]
+ )
+ total = _reduce(leaf, reducer, axis=summed)
+ group_denominator = _broadcast(total, block, divided)
+ derived.loc[:, selection] = group_denominator
+
+ denominator = derived
+ if denominators is not None:
+ # Database-computed rollups win wherever the chart requested the level;
+ # anything it did not cover falls back to the leaf-derived total.
+ denominator = denominators.combine_first(derived)
Review Comment:
Could we distinguish a missing rollup from one whose value is SQL NULL?
`combine_first` treats both as missing and replaces a real NULL denominator
with the leaf-derived total, while the renderer leaves those cells blank. The
total replacement path appears to have the same issue when it skips
`None`/`NaN` values.
##########
superset/charts/client_processing.py:
##########
@@ -75,6 +83,368 @@ def get_column_key(label: tuple[str, ...], metrics:
list[str]) -> tuple[Any, ...
return tuple(parts)
+# How a metric's rollup total is derived from its cells, mirroring
+# `additiveReducerFor` in the pivot plugin's `plugin/utilities.ts`: SUM and
+# COUNT add up, MIN takes the lowest, MAX the highest. Everything else (saved
+# metrics, adhoc SQL, AVG, ...) is non-additive and has no correct answer at
+# this layer, so it falls back to summing the cells.
+_ROLLUP_REDUCERS: dict[str, str] = {"MIN": "min", "MAX": "max"}
+DEFAULT_ROLLUP_REDUCER = "sum"
+
+
+def split_grouping_sets_levels(
+ df: pd.DataFrame,
+) -> tuple[pd.DataFrame, dict[frozenset[str], pd.DataFrame]]:
+ """
+ Separate a GROUPING SETS result into its leaf frame and rollup levels.
+
+ A pivot chart with non-additive metrics asks for every rollup level in one
+ frame, tagging each row with a ``GROUPING()`` marker per groupby column
+ (see ``common/grouping_sets.py``): ``0`` where the column is grouped at
that
+ row's level, ``1`` where it has been rolled up. The rollup rows must not be
+ pivoted as ordinary rows -- their collapsed dimensions are NULL, so they
+ would add phantom rows and columns and inflate every denominator.
+
+ They are not discardable either: for a non-additive metric the database
+ rollup is the only correct total, and re-deriving one from the leaf cells
+ gives a different number (the mean of means, say, rather than the mean).
+ The chart divides by these values, so the export has to as well.
+
+ :return: the leaf frame, and each rollup level keyed by its grouped columns
+ """
+ markers = [
+ column
+ for column in df.columns
+ if isinstance(column, str) and column.endswith(GROUPING_MARKER_SUFFIX)
+ ]
+ if not markers:
+ return df, {}
+
+ grouped_of = {marker: marker[: -len(GROUPING_MARKER_SUFFIX)] for marker in
markers}
+ levels: dict[frozenset[str], pd.DataFrame] = {}
+ leaf = df
+ for keys, rows_at_level in df.groupby(markers, sort=False):
+ # `groupby` yields a scalar key for a single column and a tuple beyond.
+ marker_values = keys if isinstance(keys, tuple) else (keys,)
+ grouped = frozenset(
+ grouped_of[marker]
+ for marker, rolled_up in zip(markers, marker_values, strict=True)
+ if not rolled_up
+ )
+ level = rows_at_level.drop(columns=markers).reset_index(drop=True)
+ levels[grouped] = level
+ if len(grouped) == len(markers):
+ leaf = level
+ return leaf, levels
+
+
+def get_metric_rollup_reducers(
+ metrics: list[Any], verbose_map: Optional[dict[str, Any]] = None
+) -> dict[str, str]:
+ """Map each metric's label to the reducer that rolls its cells up."""
+ reducers: dict[str, str] = {}
+ for metric in metrics:
+ reducer = DEFAULT_ROLLUP_REDUCER
+ if isinstance(metric, dict) and metric.get("expressionType") ==
"SIMPLE":
+ reducer = _ROLLUP_REDUCERS.get(
+ metric.get("aggregate") or "", DEFAULT_ROLLUP_REDUCER
+ )
+ reducers[get_metric_name(metric, verbose_map)] = reducer
+ return reducers
+
+
+def _collapsed_metric(present: list[Any], metrics: list[str]) -> Any:
+ """
+ The metric a rollup spanning `present` stands for.
+
+ A total that collapses the metric axis is undefined in the renderer, which
+ resolves it to the last metric pushed into the shared slot (see the
+ ``metricAxis`` handling in ``react-pivottable/utilities.ts``). Mirror that
+ by taking the last metric in the configured order, so exported percentages
+ match the chart rather than summing metrics that share no unit.
+ """
+ distinct = set(present)
+ if len(distinct) == 1:
+ return distinct.pop()
+ for metric in reversed(metrics):
+ if metric in distinct:
+ return metric
+ return None
+
+
+def _broadcast(total: pd.Series, block: pd.DataFrame, axis: int) ->
pd.DataFrame:
+ """Spread a per-row (`axis` 0) or per-column (`axis` 1) total over
`block`."""
+ if axis == 0:
+ spread = pd.concat([total] * len(block.columns), axis=1)
+ spread.columns = block.columns
+ return spread
+ return pd.DataFrame(
+ np.tile(total.reindex(block.columns).to_numpy(), (len(block.index),
1)),
+ index=block.index,
+ columns=block.columns,
+ )
+
+
+def _metric_of_column(column: Any, metric_level: int) -> Any:
+ """The metric a pivoted column belongs to."""
+ return column[metric_level] if isinstance(column, tuple) else column
+
+
+def _reduce(
+ data: Union[pd.DataFrame, pd.Series],
+ reducer: str,
+ axis: Optional[int] = None,
+) -> Any:
+ """Apply a rollup reducer (``sum``/``min``/``max``), skipping empty
cells."""
+ method = getattr(data, reducer)
+ return method(axis=axis) if axis is not None else method()
+
+
+def _keyed_rollup(
+ frame: pd.DataFrame, dimensions: list[str]
+) -> dict[tuple[str, ...], dict[str, Any]]:
+ """Index a rollup level by its grouped dimension values, as strings."""
+ filled = frame.fillna("SUPERSET_PANDAS_NAN")
+ return {
+ tuple(str(record[dimension]) for dimension in dimensions): record
+ for record in filled.to_dict("records")
+ }
+
+
+def _rollup_key(
+ label: Any, depth: int, metric_level: int, is_column: bool
+) -> tuple[str, ...]:
+ """The grouped dimension values a pivoted row or column label carries."""
+ parts = list(label) if isinstance(label, tuple) else [label]
+ if is_column:
+ # The column label interleaves the metric with the dimension values.
+ parts = [part for index, part in enumerate(parts) if index !=
metric_level]
+ return tuple(str(part) for part in parts[:depth])
+
+
+def _apply_rollup_totals( # pylint: disable=too-many-arguments,too-many-locals
+ df: pd.DataFrame,
+ rows: list[str],
+ columns: list[str],
+ metrics: list[str],
+ rollup_levels: dict[frozenset[str], pd.DataFrame],
+ metric_level: int,
+ row_prefix_depth: dict[Any, int],
+ column_prefix_depth: dict[Any, int],
+) -> pd.DataFrame:
+ """
+ Replace inserted totals with the values the database computed.
+
+ A total grouping ``i`` row dimensions and ``j`` column dimensions is
exactly
+ the rollup level over ``rows[:i] + columns[:j]``, which
+ ``buildGroupbyCombinations`` requests whenever the chart displays that
+ total. Reading it keeps the export equal to the chart for a non-additive
+ metric, where reducing the leaf cells gives a different number.
+
+ Any total whose level or key is absent keeps its leaf-derived value, so a
+ missing level degrades to the previous behaviour rather than to a blank.
+ """
+ metric_names = set(metrics)
+
+ def metric_of(column: Any) -> Any:
+ name = _metric_of_column(column, metric_level)
+ return (
+ name if name in metric_names else _collapsed_metric(list(metrics),
metrics)
+ )
+
+ def lookup(row: Any, column: Any) -> Any:
+ row_depth = row_prefix_depth.get(row, len(rows))
+ column_depth = column_prefix_depth.get(column, len(columns))
+ level = rollup_levels.get(frozenset(rows[:row_depth] +
columns[:column_depth]))
+ if level is None:
+ return None
+ key = _rollup_key(row, row_depth, metric_level, is_column=False) +
_rollup_key(
+ column, column_depth, metric_level, is_column=True
+ )
+ record = _keyed_rollup(level, rows[:row_depth] +
columns[:column_depth]).get(
+ key
+ )
+ return record.get(metric_of(column)) if record else None
+
+ # Index positionally: a tuple label on a MultiIndex is ambiguous to `.loc`,
+ # which resolves some of these keys to the wrong row.
+ for column_position, column in enumerate(df.columns):
+ for row_position, row in enumerate(df.index):
+ if row not in row_prefix_depth and column not in
column_prefix_depth:
+ continue # a leaf cell, already carrying its own value
+ value = lookup(row, column)
+ if value is not None and not pd.isna(value):
+ df.iloc[row_position, column_position] = value
+ return df
+
+
+def _rollup_denominators( # pylint: disable=too-many-arguments,too-many-locals
+ df: pd.DataFrame,
+ mode: str,
+ rows: list[str],
+ columns: list[str],
+ metrics: list[str],
+ rollup_levels: dict[frozenset[str], pd.DataFrame],
+ metric_level: int,
+ row_prefix_depth: dict[Any, int],
+ column_prefix_depth: dict[Any, int],
+) -> Optional[pd.DataFrame]:
+ """
+ Each cell's denominator, taken from the database-computed rollup levels.
+
+ A percent mode makes the chart request the level its denominator needs: for
+ "% of row" the cell's own row with the columns collapsed, for "% of column"
+ the reverse, for "% of total" both (see ``buildGroupbyCombinations``). A
+ subtotal divides by its own prefix, not by the grand total -- an "EU"
+ subtotal row divides by the ``{region}`` rollup.
+
+ For a non-additive metric these are the only correct totals. Cells whose
+ level or key is absent come back NaN, leaving the caller to fall back to a
+ denominator derived from the leaf cells.
+ """
+ if not rollup_levels:
+ return None
+ metric_names = set(metrics)
+
+ def metric_of(column: Any) -> Any:
+ name = _metric_of_column(column, metric_level)
+ return (
+ name if name in metric_names else _collapsed_metric(list(metrics),
metrics)
+ )
+
+ def denominator(row: Any, column: Any) -> Any:
+ row_depth = row_prefix_depth.get(row, len(rows))
+ column_depth = column_prefix_depth.get(column, len(columns))
+ if mode == ShowValuesAs.PERCENT_OF_ROW:
+ grouped, key = (
+ rows[:row_depth],
+ _rollup_key(row, row_depth, metric_level, is_column=False),
+ )
+ elif mode == ShowValuesAs.PERCENT_OF_COLUMN:
+ grouped, key = (
+ columns[:column_depth],
+ _rollup_key(column, column_depth, metric_level,
is_column=True),
+ )
+ else:
+ grouped, key = [], ()
+ level = rollup_levels.get(frozenset(grouped))
+ if level is None:
+ return None
+ record = _keyed_rollup(level, grouped).get(key)
Review Comment:
Could we build the keyed rollup maps once per level? This runs inside the
per-cell loop, and `_keyed_rollup` repeats `fillna()` plus `to_dict("records")`
each time. A 200×50 pivot spent about 3.8s here locally; caching the maps
before the loop should make this effectively linear.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
