aminghadersohi commented on code in PR #44739:
URL: https://github.com/apache/superset/pull/44739#discussion_r4129983156
##########
tests/unit_tests/dataframe_test.py:
##########
@@ -363,3 +365,44 @@ def
test_df_to_records_with_json_serialization_like_sql_lab() -> None:
)
parsed_no_flag = superset_json.loads(json_str_no_flag)
assert parsed_no_flag == parsed # Same result
+
+
[email protected](
+ "value",
+ [
+ "12345678901234567890.123456789012345678",
+ "-12345678901234567890.123456789012345678",
+ "0.000000000000000001",
+ "10.50",
+ "0.00",
+ "1E+30",
+ ],
+)
+def test_decimal_records_keep_all_digits(value: str) -> None:
Review Comment:
Added one-line docstrings to all the new decimal tests (and the new
chart-records test) in cd7c69f6e2.
##########
tests/unit_tests/dataframe_test.py:
##########
@@ -363,3 +365,44 @@ def
test_df_to_records_with_json_serialization_like_sql_lab() -> None:
)
parsed_no_flag = superset_json.loads(json_str_no_flag)
assert parsed_no_flag == parsed # Same result
+
+
[email protected](
+ "value",
+ [
+ "12345678901234567890.123456789012345678",
+ "-12345678901234567890.123456789012345678",
+ "0.000000000000000001",
+ "10.50",
+ "0.00",
+ "1E+30",
+ ],
+)
+def test_decimal_records_keep_all_digits(value: str) -> None:
+ decimal = Decimal(value)
Review Comment:
Renamed the local to `decimal_value: Decimal` in cd7c69f6e2.
##########
superset/dataframe.py:
##########
@@ -72,7 +86,9 @@ def df_to_records(dframe: pd.DataFrame) -> list[dict[str,
Any]]:
for record in records:
for key in record:
record[key] = (
- None if _is_na(record[key]) else
_convert_big_integers(record[key])
+ None
+ if _is_na(record[key])
+ else _convert_decimals(_convert_big_integers(record[key]))
)
Review Comment:
Rechecked after merging master (cd7c69f6e2); the finding still doesn't
apply. `superset/sql/execution/executor.py:1012` hands the `to_dict` records to
`cache_manager.data_cache.set`, and `data_cache` is `SupersetCache`
(`superset/utils/cache_manager.py:82`). That class is a Flask-Caching `Cache`
subclass that only overrides key hashing. Flask-Caching backends pickle their
values, so a `Decimal` comes back from `_get_from_cache` as the same `Decimal`,
and no JSON step drops digits. SQL Lab results, which is what this PR changes,
go through `df_to_records` in `sql_lab.py` / `sql/execution/celery_task.py`.
##########
superset-frontend/src/components/FilterableTable/sortResults.ts:
##########
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+type CellValue = string | number | null;
+
+// Compare significands and decimal orders, never coerce exact strings to
Number.
+// Keeping the exponent separate also avoids allocating 10**exponent zeroes.
+const DECIMAL = /^([+-]?)(?:(\d+)(?:\.(\d*))?|\.(\d+))(?:[eE]([+-]?\d+))?$/;
+
+function decimalParts(value: Exclude<CellValue, null>) {
+ const match = DECIMAL.exec(String(value));
+ if (!match) return null;
+ const fraction = match[3] ?? match[4] ?? '';
+ const digits = `${match[2] ?? ''}${fraction}`.replace(/^0+/, '');
+ return {
+ sign: digits ? (match[1] === '-' ? -1 : 1) : 0,
+ digits,
+ order:
+ BigInt(digits.length) + BigInt(match[5] ?? '0') -
BigInt(fraction.length),
+ };
+}
+
+export function sortResults(valueA: CellValue, valueB: CellValue): number {
+ if (valueA === valueB) return 0;
+ if (valueA === null) return 1;
+ if (valueB === null) return -1;
+
+ const a = decimalParts(valueA);
+ const b = decimalParts(valueB);
+ if (a && b) {
+ if (a.sign !== b.sign) return a.sign < b.sign ? -1 : 1;
+ if (a.sign === 0) return 0;
+ if (a.order !== b.order) return (a.order < b.order ? -1 : 1) * a.sign;
+ const width = Math.max(a.digits.length, b.digits.length);
+ const left = a.digits.padEnd(width, '0');
+ const right = b.digits.padEnd(width, '0');
+ return left === right ? 0 : (left < right ? -1 : 1) * a.sign;
+ }
+
+ // Retain the table's existing numeric-string, text and infinity behavior.
+ const numberOrText = (value: Exclude<CellValue, null>) =>
+ typeof value === 'string' &&
+ /^(NaN|-?((\d*\.\d+|\d+)([Ee][+-]?\d+)?|Infinity))$/.test(value)
+ ? Number(value)
+ : value;
+ const left = numberOrText(valueA);
+ const right = numberOrText(valueB);
+ return left === right ? 0 : left < right ? -1 : 1;
Review Comment:
Verified at the current head (394a03dd28): the fallback comparison
introduced in 06ed8bc4ee is unchanged, and the antisymmetry and transitivity
tests still pass.
##########
superset-frontend/src/components/FilterableTable/sortResults.ts:
##########
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+type CellValue = string | number | null;
+
+// Compare significands and decimal orders, never coerce exact strings to
Number.
+// Keeping the exponent separate also avoids allocating 10**exponent zeroes.
+const DECIMAL = /^([+-]?)(?:(\d+)(?:\.(\d*))?|\.(\d+))(?:[eE]([+-]?\d+))?$/;
+
+function decimalParts(value: Exclude<CellValue, null>) {
+ const match = DECIMAL.exec(String(value));
+ if (!match) return null;
+ const fraction = match[3] ?? match[4] ?? '';
+ const digits = `${match[2] ?? ''}${fraction}`.replace(/^0+/, '');
+ return {
+ sign: digits ? (match[1] === '-' ? -1 : 1) : 0,
+ digits,
+ order:
+ BigInt(digits.length) + BigInt(match[5] ?? '0') -
BigInt(fraction.length),
+ };
+}
+
+export function sortResults(valueA: CellValue, valueB: CellValue): number {
+ if (valueA === valueB) return 0;
+ if (valueA === null) return 1;
+ if (valueB === null) return -1;
+
+ const a = decimalParts(valueA);
+ const b = decimalParts(valueB);
+ if (a && b) {
+ if (a.sign !== b.sign) return a.sign < b.sign ? -1 : 1;
+ if (a.sign === 0) return 0;
+ if (a.order !== b.order) return (a.order < b.order ? -1 : 1) * a.sign;
+ const width = Math.max(a.digits.length, b.digits.length);
+ const left = a.digits.padEnd(width, '0');
+ const right = b.digits.padEnd(width, '0');
+ return left === right ? 0 : (left < right ? -1 : 1) * a.sign;
+ }
+
+ // Retain the table's existing numeric-string, text and infinity behavior.
+ const numberOrText = (value: Exclude<CellValue, null>) =>
+ typeof value === 'string' &&
+ /^(NaN|-?((\d*\.\d+|\d+)([Ee][+-]?\d+)?|Infinity))$/.test(value)
+ ? Number(value)
+ : value;
+ const left = numberOrText(valueA);
+ const right = numberOrText(valueB);
+ return left === right ? 0 : left < right ? -1 : 1;
Review Comment:
Verified at the current head (394a03dd28): the fix from 06ed8bc4ee is
unchanged, and the NaN and mixed-type cases in sortResults.test.ts still pass.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
