This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 0592b92168 [Python] Align decimal schema evolution with Java and
Native (#10531)
0592b92168 is described below
commit 0592b921686bd8a8c3c5aa0cdc0227a50d9b6018
Author: Jingsong Lee <[email protected]>
AuthorDate: Tue Oct 13 14:47:44 2026 +0800
[Python] Align decimal schema evolution with Java and Native (#10531)
---
paimon-python/pypaimon/casting/decimal_cast.py | 81 +++++++++++
paimon-python/pypaimon/casting/numeric_utils.py | 150 +++++++++++++++++++++
paimon-python/pypaimon/casting/schema_evolution.py | 21 +--
.../pypaimon/tests/native_decimal_cast_test.py | 143 ++++++++++++++++++++
paimon-python/pypaimon/tests/numeric_utils_test.py | 75 +++++++++++
.../pypaimon/tests/schema_evolution_cast_test.py | 94 ++++++++++++-
.../pypaimon/utils/file_store_path_factory.py | 41 +-----
7 files changed, 550 insertions(+), 55 deletions(-)
diff --git a/paimon-python/pypaimon/casting/decimal_cast.py
b/paimon-python/pypaimon/casting/decimal_cast.py
new file mode 100644
index 0000000000..eb7170dbea
--- /dev/null
+++ b/paimon-python/pypaimon/casting/decimal_cast.py
@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Java decimal schema evolution: decimal text, HALF_UP, then precision."""
+
+from decimal import Context, Decimal, InvalidOperation, localcontext,
ROUND_HALF_UP
+import re
+
+import pyarrow as pa
+
+from pypaimon.casting.numeric_utils import java_double_decimal_text
+
+
+_DECIMAL_TEXT =
re.compile(r'[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?\Z')
+
+
+def cast_decimal(array, target):
+ """Return None for conversions outside the existing decimal read support.
+
+ FLOAT values from Arrow are already widened to Python double. Java does
+ the same widening before BigDecimal.valueOf, retaining the float's binary
+ rounding. A valid decimal overflowing the target is NULL; invalid text
+ fails the conversion instead of silently becoming NULL.
+ """
+ floating = pa.types.is_floating(array.type)
+ text = pa.types.is_string(array.type)
+ if not pa.types.is_decimal(target) or not (floating or text or
pa.types.is_decimal(array.type)):
+ return None
+ values = []
+ # The caller's rounding traps and exponent limits must not affect reads.
+ # One extra digit allows rounding to carry, before the precision check.
+ context = Context(prec=target.precision + 1, rounding=ROUND_HALF_UP,
+ Emin=-999999, Emax=999999, clamp=0,
traps=[InvalidOperation])
+ with localcontext(context):
+ quantum = Decimal(1).scaleb(-target.scale)
+ for value in array.to_pylist():
+ if value is None:
+ values.append(None)
+ continue
+ if floating:
+ value = java_double_decimal_text(value)
+ if floating or text:
+ if _DECIMAL_TEXT.fullmatch(value) is None:
+ raise pa.ArrowInvalid('Invalid decimal schema-evolution
value: %s' % value)
+ try:
+ value = Decimal(value)
+ except InvalidOperation as error:
+ raise pa.ArrowInvalid('Invalid decimal schema-evolution
value: %s' % value) from error
+ # BigDecimal validates the adjusted input scale, then setScale and
+ # BigInteger.TEN.pow check their supported range before precision.
+ # Zero bypasses the power check, but still needs a valid input
scale.
+ input_scale = -value.as_tuple().exponent
+ if not -(2 ** 31) <= input_scale < 2 ** 31:
+ raise pa.ArrowInvalid('Invalid decimal schema-evolution input
scale')
+ power = abs(target.scale - input_scale)
+ if not value.is_zero() and (power > 2 ** 31 - 1 or 4 * power // 32
> (2 ** 31 - 1) // 32 + 1):
+ raise pa.ArrowInvalid("Decimal scale conversion exceeds Java's
supported range")
+ # Avoid quantizing values whose integer part already overflows.
+ if not value.is_zero() and value.adjusted() + target.scale + 1 >
target.precision:
+ values.append(None)
+ continue
+ try:
+ rounded = value.quantize(quantum, rounding=ROUND_HALF_UP)
+ except InvalidOperation as error:
+ raise pa.ArrowInvalid('Invalid decimal schema-evolution value:
%s' % value) from error
+ values.append(rounded if len(rounded.as_tuple().digits) <=
target.precision else None)
+ return pa.array(values, type=target)
diff --git a/paimon-python/pypaimon/casting/numeric_utils.py
b/paimon-python/pypaimon/casting/numeric_utils.py
new file mode 100644
index 0000000000..a44ec07cfa
--- /dev/null
+++ b/paimon-python/pypaimon/casting/numeric_utils.py
@@ -0,0 +1,150 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Java numeric text shared by partition paths and value conversions."""
+
+import math
+import struct
+from decimal import Decimal
+
+
+def java_floating_string(value, single_precision=False) -> str:
+ # Java's JDK 19+ contract selects a shortest round-tripping form with at
+ # least two significant digits before applying its notation thresholds.
+ encoding = '>f' if single_precision else '>d'
+ bits = struct.pack(encoding, value)
+ value = struct.unpack(encoding, bits)[0]
+ if math.isnan(value):
+ return 'NaN'
+ if math.isinf(value):
+ return '-Infinity' if value < 0 else 'Infinity'
+ for precision in range(2, 10 if single_precision else 18):
+ text = format(value, '.{}g'.format(precision))
+ try:
+ rounded = struct.pack(encoding, float(text))
+ except OverflowError:
+ continue
+ if rounded != bits:
+ continue
+ decimal = Decimal(text)
+ if decimal.is_zero() or Decimal('0.001') <= abs(decimal) <
Decimal('1e7'):
+ text = format(decimal, 'f')
+ if '.' in text:
+ text = text.rstrip('0').rstrip('.')
+ if '.' not in text:
+ text += '.0'
+ else:
+ mantissa, exponent = format(decimal, 'e').split('e')
+ mantissa = mantissa.rstrip('0').rstrip('.') if '.' in mantissa
else mantissa
+ if '.' not in mantissa:
+ mantissa += '.0'
+ text = '{}E{}'.format(mantissa, int(exponent))
+ return text
+ return str(value)
+
+
+def java_double_decimal_text(value):
+ """Numeric digits of Java 8/11/17 Double.toString for BigDecimal.valueOf.
+
+ The pre-JDK-19 converter has an integral fast path and distinct interval
+ endpoints for its fixed-width and arbitrary-precision digit generators.
+ Use integer arithmetic over IEEE-754 components to preserve those choices.
+ The emitted coefficient/exponent is numeric text, not a display format.
+ """
+ if not math.isfinite(value):
+ return java_floating_string(value)
+ if value == 0:
+ return '0'
+ raw = struct.unpack('>Q', struct.pack('>d', value))[0]
+ sign = '-' if raw >> 63 else ''
+ exponent = ((raw >> 52) & 2047) - 1023
+ mantissa = raw & ((1 << 52) - 1)
+ if exponent == -1023:
+ significant = mantissa.bit_length()
+ normalize = 53 - significant
+ mantissa <<= normalize
+ exponent = -1022 - normalize
+ else:
+ significant = 53
+ mantissa |= 1 << 52
+ trailing = (mantissa & -mantissa).bit_length() - 1
+ width = 53 - trailing
+ tiny = max(0, width - exponent - 1)
+ if -21 <= exponent <= 62 and tiny == 0:
+ integer = mantissa << (exponent - 52) if exponent >= 52 else mantissa
>> (52 - exponent)
+ discarded = (len(str(1 << max(0, exponent - significant - 1))) - 1
+ if exponent > significant else 0)
+ factor = 10 ** discarded
+ return sign + str((integer + factor // 2) // factor) + 'e' +
str(discarded)
+
+ # Approximate log10 from the normalized binary significand. A leading zero
+ # digit corrects an overestimate without changing the generated interval.
+ normalized = mantissa / (1 << 52)
+ decimal_exponent = math.floor(
+ (normalized - 1.5) * 0.289529654 + 0.176091259 + exponent *
0.301029995663981)
+ negative_power = max(0, -decimal_exponent)
+ positive_power = max(0, decimal_exponent)
+ numerator_power = negative_power + tiny + exponent - width + 1
+ denominator_power = positive_power + tiny
+ margin_power = negative_power + tiny + exponent - significant
+ common = min(numerator_power, denominator_power)
+ numerator_power -= common
+ denominator_power -= common
+ margin_power -= common
+ if width == 1:
+ margin_power -= 1
+ if margin_power < 0:
+ numerator_power -= margin_power
+ denominator_power -= margin_power
+ margin_power = 0
+ numerator = (mantissa >> trailing) * 5 ** negative_power * (1 <<
numerator_power)
+ denominator = 5 ** positive_power * (1 << denominator_power)
+ margin = 5 ** negative_power * (1 << margin_power)
+
+ def five_bits(power):
+ return (5 ** power).bit_length() if 0 < power < 27 else power * 3
+
+ bits = max(width + numerator_power + five_bits(negative_power),
+ denominator_power + 1 + five_bits(positive_power + 1))
+ word = 32 if bits < 32 else 64 if bits < 64 else 0
+
+ def wrap(number):
+ return (number + (1 << (word - 1))) % (1 << word) - (1 << (word - 1))
if word else number
+
+ ten_denominator = denominator * 10
+ coefficient, count = 0, 0
+ while True:
+ digit, numerator = divmod(numerator, denominator)
+ numerator *= 10
+ margin = wrap(margin * 10)
+ low = numerator < margin
+ high = (wrap(numerator + margin) > ten_denominator if word else
+ numerator + margin >= ten_denominator)
+ if count and word and margin <= 0:
+ low = high = True
+ if digit == 0 and count == 0 and not high:
+ decimal_exponent -= 1
+ else:
+ coefficient = coefficient * 10 + digit
+ count += 1
+ if count <= 1 and (decimal_exponent < -3 or decimal_exponent >= 8):
+ low = high = False
+ if low or high:
+ difference = wrap(2 * numerator - ten_denominator)
+ if high and (not low or difference > 0 or difference == 0 and
coefficient % 2):
+ coefficient += 1
+ return sign + str(coefficient) + 'e' + str(decimal_exponent -
count + 1)
diff --git a/paimon-python/pypaimon/casting/schema_evolution.py
b/paimon-python/pypaimon/casting/schema_evolution.py
index afa3650627..1116aeaacd 100644
--- a/paimon-python/pypaimon/casting/schema_evolution.py
+++ b/paimon-python/pypaimon/casting/schema_evolution.py
@@ -17,11 +17,11 @@
"""Java cast rules shared by old-file readers."""
-from decimal import Decimal, localcontext, ROUND_HALF_UP
import math
import pyarrow as pa
+from pypaimon.casting.decimal_cast import cast_decimal
from pypaimon.casting.temporal_cast import cast_temporal
from pypaimon.casting.temporal_string import temporal_to_string
from pypaimon.table.row.generic_row import _parse_type_precision_scale
@@ -122,7 +122,8 @@ def cast_array_for_schema_evolution(array, target_type,
source_paimon_type=None,
epoch seconds, including pre-epoch flooring and Java's millis
multiplication.
Numeric narrowing preserves low bits, while floating-point values truncate
and saturate to int/long before byte/short narrowing.
- DECIMAL scale reduction uses HALF_UP and returns NULL on precision
overflow.
+ DECIMAL conversion uses Java decimal text and HALF_UP, returning NULL on
+ precision overflow after rounding.
LTZ conversions use the process local zone, including Java gap/overlap
rules.
"""
if pa.types.is_string(target_type):
@@ -137,6 +138,9 @@ def cast_array_for_schema_evolution(array, target_type,
source_paimon_type=None,
return timestamp
if array.type == target_type:
return array
+ decimals = cast_decimal(array, target_type)
+ if decimals is not None:
+ return decimals
if pa.types.is_timestamp(array.type) and target_type == pa.int64():
return _timestamp_to_bigint(array)
if (array.type in (pa.int32(), pa.int64())
@@ -145,17 +149,4 @@ def cast_array_for_schema_evolution(array, target_type,
source_paimon_type=None,
if (pa.types.is_signed_integer(target_type)
and (pa.types.is_floating(array.type) or
pa.types.is_decimal(array.type))):
return _numeric_to_integer(array, target_type)
- if (pa.types.is_decimal(array.type) and pa.types.is_decimal(target_type)
- and array.type.scale > target_type.scale):
- values = []
- with localcontext() as context:
- context.prec = max(array.type.precision, target_type.precision) + 1
- quantum = Decimal(1).scaleb(-target_type.scale)
- for value in array.to_pylist():
- if value is None:
- values.append(None)
- continue
- rounded = value.quantize(quantum, rounding=ROUND_HALF_UP)
- values.append(rounded if len(rounded.as_tuple().digits) <=
target_type.precision else None)
- return pa.array(values, type=target_type)
return array.cast(target_type, safe=False)
diff --git a/paimon-python/pypaimon/tests/native_decimal_cast_test.py
b/paimon-python/pypaimon/tests/native_decimal_cast_test.py
new file mode 100644
index 0000000000..7c1c998438
--- /dev/null
+++ b/paimon-python/pypaimon/tests/native_decimal_cast_test.py
@@ -0,0 +1,143 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Java decimal golden values through Python and Native REST reads."""
+
+from decimal import Decimal
+import struct
+
+import pyarrow as pa
+import pytest
+
+from pypaimon import Schema
+from pypaimon.read.native_plan import native_runtime_available
+from pypaimon.schema.data_types import AtomicType, PyarrowFieldParser
+from pypaimon.schema.schema_change import SchemaChange
+from pypaimon.tests import native_plan_rest_test
+from pypaimon.tests.native_schema_evolution_cast_test import _read, _table,
_write
+
+rest_catalog = native_plan_rest_test.rest_catalog
+
+
+pytestmark = [pytest.mark.native_plan, pytest.mark.skipif(
+ not native_runtime_available(), reason='Rust reader required')]
+
+
+_CASES = [
+ (pa.float64(), 'DECIMAL(38,20)',
+ [struct.unpack('>d', bytes.fromhex('c381e88f38c0c8fd'))[0], None],
+ ['-161304845894623136.00000000000000000000', None]),
+ (pa.float64(), 'DECIMAL(38,6)', [1e23, -1e23, None],
+ ['99999999999999990000000.000000', '-99999999999999990000000.000000',
None]),
+ (pa.float64(), 'DECIMAL(5,2)', [1.005, -1.005, 0.145, 999.995, 1e30, None],
+ ['1.01', '-1.01', '0.15', None, None, None]),
+ (pa.float32(), 'DECIMAL(5,2)', [1.005, -1.005, 0.145, 999.995, 1e30, None],
+ ['1.00', '-1.00', '0.14', '999.99', None, None]),
+ (pa.float64(), 'DECIMAL(38,20)', [1.2345678901234567, 1.005, 1e30, None],
+ ['1.23456789012345670000', '1.00500000000000000000', None, None]),
+ (pa.float32(), 'DECIMAL(38,20)', [1.2345679, 1.005, 1e30, None],
+ ['1.23456788063049320000', '1.00499999523162840000', None, None]),
+ (pa.string(), 'DECIMAL(5,2)', ['1e-2', '+.005', '2.675', '999.995',
'1e100', None],
+ ['0.01', '0.01', '2.68', None, None, None]),
+ (pa.decimal128(10, 2), 'DECIMAL(3,2)', [Decimal('9.99'), Decimal('10.00'),
None],
+ ['9.99', None, None]),
+ (pa.decimal128(10, 2), 'DECIMAL(4,3)', [Decimal('9.99'),
Decimal('-10.00'), None],
+ ['9.990', None, None]),
+ (pa.decimal128(6, 3), 'DECIMAL(3,2)', [Decimal('9.994'), Decimal('9.995'),
Decimal('-9.995'), None],
+ ['9.99', None, None, None]),
+]
+
+
[email protected]('mode', ['append', 'pk', 'de'])
[email protected]('source,target,values,expected', _CASES)
+def test_decimal_schema_changes_read_mixed_rest_files(rest_catalog, mode,
source, target, values, expected):
+ catalog, _ = rest_catalog
+ _table(catalog, mode, source, values)
+ catalog.alter_table('default.t', [SchemaChange.update_column_type('v',
AtomicType(target))], False)
+ table = catalog.get_table('default.t')
+ target_arrow =
PyarrowFieldParser.from_paimon_type(table.field_dict['v'].type)
+ expected = [None if value is None else Decimal(value) for value in
expected]
+ # Read both old and new schema files, with a batch size smaller than the
input.
+ _write(table, pa.table({'id': [len(values)], 'v': pa.array([expected[0]],
type=target_arrow)}))
+ table = catalog.get_table('default.t')
+ pb = table.new_read_builder().new_predicate_builder()
+ for native in (False, True):
+ actual = _read(table, native).column('v')
+ assert actual.type == target_arrow
+ assert actual.to_pylist() == expected + [expected[0]]
+ # The converted column remains a residual-filter input when projected
out.
+ assert _read(table, native, pb.equal('v', expected[0]),
['id']).column('id').to_pylist() == [
+ i for i, value in enumerate(expected + [expected[0]]) if value ==
expected[0]]
+ assert _read(table, native, pb.is_null('v'),
['id']).column('id').to_pylist() == [
+ i for i, value in enumerate(expected) if value is None]
+
+
[email protected]('source,target,values,expected', _CASES)
+def
test_decimal_single_value_stats_do_not_prune_converted_matches(rest_catalog,
source, target, values, expected):
+ catalog, _ = rest_catalog
+ _table(catalog, 'append', source, values[:1])
+ catalog.alter_table('default.t', [SchemaChange.update_column_type('v',
AtomicType(target))], False)
+ table = catalog.get_table('default.t')
+ predicate = table.new_read_builder().new_predicate_builder().equal('v',
Decimal(expected[0]))
+ for native in (False, True):
+ assert _read(table, native, predicate,
['id']).column('id').to_pylist() == [0]
+
+
[email protected]('mode', ['append', 'pk', 'de'])
+def test_nested_decimal_casts_preserve_containers_and_projection(rest_catalog,
mode):
+ catalog, _ = rest_catalog
+ schema = pa.schema([
+ ('id', pa.int64()), ('r', pa.struct([('v', pa.float32())])),
+ ('a', pa.list_(pa.float64())), ('m', pa.map_(pa.string(),
pa.string())),
+ ])
+ options = {'file.format': 'parquet', 'read.batch-size': '1'}
+ if mode == 'pk':
+ options['bucket'] = '1'
+ elif mode == 'de':
+ options.update({'data-evolution.enabled': 'true',
'row-tracking.enabled': 'true'})
+ catalog.create_table('default.t', Schema.from_pyarrow_schema(
+ schema, primary_keys=['id'] if mode == 'pk' else [], options=options),
False)
+ _write(catalog.get_table('default.t'), pa.Table.from_pylist([
+ {'id': 0, 'r': {'v': 1.2345679}, 'a': [1.005, -1.005, 999.995, None],
+ 'm': [('one', '1e-2'), ('two', '+.005'), ('big', '1e100')]},
+ {'id': 1, 'r': None, 'a': None, 'm': None},
+ {'id': 2, 'r': {'v': None}, 'a': [], 'm': []},
+ ], schema=schema))
+ catalog.alter_table('default.t', [
+ SchemaChange.update_column_type(['r', 'v'],
AtomicType('DECIMAL(38,20)')),
+ SchemaChange.update_column_type(['a', 'element'],
AtomicType('DECIMAL(5,2)')),
+ SchemaChange.update_column_type(['m', 'value'],
AtomicType('DECIMAL(5,2)')),
+ ], False)
+ table = catalog.get_table('default.t')
+ for native in (False, True):
+ actual = _read(table, native)
+ assert actual.column('r').to_pylist() == [{'v':
Decimal('1.23456788063049320000')}, None, {'v': None}]
+ assert actual.column('a').to_pylist() == [[Decimal('1.01'),
Decimal('-1.01'), None, None], None, []]
+ assert actual.column('m').to_pylist() == [[('one', Decimal('0.01')),
('two', Decimal('0.01')),
+ ('big', None)], None, []]
+ assert _read(table, native, projection=['id',
'r.v']).column('r_v').to_pylist() == [
+ Decimal('1.23456788063049320000'), None, None]
+
+
[email protected]('mode', ['append', 'pk', 'de'])
+def test_invalid_decimal_old_file_fails_in_both_readers(rest_catalog, mode):
+ catalog, _ = rest_catalog
+ _table(catalog, mode, pa.string(), [None, '1.2.3'])
+ catalog.alter_table('default.t', [SchemaChange.update_column_type('v',
AtomicType('DECIMAL(5,2)'))], False)
+ for native in (False, True):
+ with pytest.raises(Exception, match='Invalid decimal schema-evolution
value'):
+ _read(catalog.get_table('default.t'), native)
diff --git a/paimon-python/pypaimon/tests/numeric_utils_test.py
b/paimon-python/pypaimon/tests/numeric_utils_test.py
new file mode 100644
index 0000000000..3ef464c0fa
--- /dev/null
+++ b/paimon-python/pypaimon/tests/numeric_utils_test.py
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Golden numeric digits from Java 8, 11 and 17 Double.toString."""
+
+from decimal import Decimal
+import struct
+
+import pytest
+
+from pypaimon.casting.numeric_utils import java_double_decimal_text
+
+
[email protected]('bits,expected', [
+ (0xc381e88f38c0c8fd, '-1.61304845894623136E17'),
+ (0x44b52d02c7e14af6, '9.999999999999999E22'),
+ (0x0000000000000001, '4.9E-324'),
+ (0x0010000000000000, '2.2250738585072014E-308'),
+ (0x7fefffffffffffff, '1.7976931348623157E308'),
+ (0x0000000000000002, '1.0E-323'),
+ (0x000fffffffffffff, '2.225073858507201E-308'),
+ (0x0010000000000001, '2.225073858507202E-308'),
+ (0x016fffffffffffff, '9.332636185032188E-302'),
+ (0x0170000000000000, '9.332636185032189E-302'),
+ (0x0170000000000001, '9.33263618503219E-302'),
+ (0x39afffffffffffff, '7.888609052210117E-31'),
+ (0x39b0000000000000, '7.888609052210118E-31'),
+ (0x39b0000000000001, '7.88860905221012E-31'),
+ (0x3befffffffffffff, '5.4210108624275216E-20'),
+ (0x3bf0000000000000, '5.421010862427522E-20'),
+ (0x3bf0000000000001, '5.421010862427523E-20'),
+ (0x3c9fffffffffffff, '1.1102230246251564E-16'),
+ (0x3ca0000000000000, '1.1102230246251565E-16'),
+ (0x3ca0000000000001, '1.1102230246251568E-16'),
+ (0x3e9fffffffffffff, '4.7683715820312495E-7'),
+ (0x3ea0000000000000, '4.76837158203125E-7'),
+ (0x3ea0000000000001, '4.768371582031251E-7'),
+ (0x3fbfffffffffffff, '0.12499999999999999'),
+ (0x3fc0000000000000, '0.125'),
+ (0x3fc0000000000001, '0.12500000000000003'),
+ (0x3fefffffffffffff, '0.9999999999999999'),
+ (0x3ff0000000000000, '1.0'),
+ (0x3ff0000000000001, '1.0000000000000002'),
+ (0x3fffffffffffffff, '1.9999999999999998'),
+ (0x4000000000000000, '2.0'),
+ (0x4000000000000001, '2.0000000000000004'),
+ (0x41efffffffffffff, '4.2949672959999995E9'),
+ (0x41f0000000000000, '4.294967296E9'),
+ (0x41f0000000000001, '4.294967296000001E9'),
+ (0x432fffffffffffff, '4.5035996273704955E15'),
+ (0x4330000000000000, '4.503599627370496E15'),
+ (0x4330000000000001, '4.503599627370497E15'),
+ (0x433fffffffffffff, '9.007199254740991E15'),
+ (0x4340000000000000, '9.007199254740992E15'),
+ (0x4340000000000001, '9.007199254740994E15'),
+ (0x434fffffffffffff, '1.8014398509481982E16'),
+ (0x4350000000000000, '1.8014398509481984E16'),
+])
+def test_double_decimal_digits_match_supported_jvms(bits, expected):
+ value = struct.unpack('>d', struct.pack('>Q', bits))[0]
+ assert Decimal(java_double_decimal_text(value)) == Decimal(expected)
diff --git a/paimon-python/pypaimon/tests/schema_evolution_cast_test.py
b/paimon-python/pypaimon/tests/schema_evolution_cast_test.py
index f3280ec9f8..363986c77d 100644
--- a/paimon-python/pypaimon/tests/schema_evolution_cast_test.py
+++ b/paimon-python/pypaimon/tests/schema_evolution_cast_test.py
@@ -15,8 +15,9 @@
# specific language governing permissions and limitations
# under the License.
-from decimal import Decimal
+from decimal import Decimal, DefaultContext, Inexact, localcontext, Rounded
import os
+import struct
import time
from types import SimpleNamespace
@@ -27,6 +28,97 @@ from pypaimon.casting.schema_evolution import
cast_array_for_schema_evolution
from pypaimon.schema.data_types import AtomicType
[email protected]('chunked', [False, True])
+def test_decimal_evolution_uses_supported_jvm_floating_decimal_digits(chunked):
+ # Java 8/11/17 retain exact digits in this integral fast path. JDK 19+
+ # shortest text changes the integer by four, before scale is applied.
+ value = struct.unpack('>d', bytes.fromhex('c381e88f38c0c8fd'))[0]
+ source = pa.array([None, value, -value, 1e23, None]).slice(1, 4)
+ if chunked:
+ source = pa.chunked_array([source.slice(0, 1), source.slice(1)])
+ result = cast_array_for_schema_evolution(source, pa.decimal128(38, 20))
+ assert result.to_pylist() == [
+ Decimal('-161304845894623136.00000000000000000000'),
+ Decimal('161304845894623136.00000000000000000000'), None, None,
+ ]
+ # For the power-of-ten boundary, the old converter excludes the upper
+ # interval endpoint and preserves 9.999999999999999E22, rather than 1E23.
+ result = cast_array_for_schema_evolution(pa.array([1e23]),
pa.decimal128(38, 6))
+ assert result.to_pylist() == [Decimal('99999999999999990000000.000000')]
+
+
[email protected]('chunked', [False, True])
[email protected]('source_type,values,precision,scale,expected', [
+ (pa.float64(), [1.005, -1.005, 2.675, -2.675, 0.145, 999.995, 1e30, None],
5, 2,
+ ['1.01', '-1.01', '2.68', '-2.68', '0.15', None, None, None]),
+ (pa.float32(), [1.005, -1.005, 2.675, 0.145, 999.995, 1e30, None], 5, 2,
+ ['1.00', '-1.00', '2.67', '0.14', '999.99', None, None]),
+ (pa.float64(), [1.005, 1.2345678901234567, 2.2250738585072014e-308, None],
38, 20,
+ ['1.00500000000000000000', '1.23456789012345670000',
'0.00000000000000000000', None]),
+ (pa.float32(), [1.005, 1.2345679, None], 38, 20,
+ ['1.00499999523162840000', '1.23456788063049320000', None]),
+ (pa.string(), ['1.005', '-1.005', '+.005', '2.675', '1e-2', '999.995',
'1e100', None], 5, 2,
+ ['1.01', '-1.01', '0.01', '2.68', '0.01', None, None, None]),
+ (pa.decimal128(10, 2), [Decimal('9.99'), Decimal('10.00'),
Decimal('-10.00'), None], 3, 2,
+ ['9.99', None, None, None]),
+ (pa.decimal128(10, 2), [Decimal('9.99'), Decimal('10.00'),
Decimal('-10.00'), None], 4, 3,
+ ['9.990', None, None, None]),
+ (pa.decimal128(6, 3), [Decimal('9.994'), Decimal('9.995'),
Decimal('-9.995'), None], 3, 2,
+ ['9.99', None, None, None]),
+])
+def test_decimal_evolution_matches_java_rounding_and_precision(
+ chunked, source_type, values, precision, scale, expected):
+ # Golden values from CastExecutors: FLOAT is promoted to double before
+ # BigDecimal.valueOf; overflow is NULL after HALF_UP, never wrapping.
+ source = pa.array([None] + values + [None], type=source_type).slice(1,
len(values))
+ if chunked:
+ source = pa.chunked_array([source.slice(0, 2), source.slice(2, 0),
source.slice(2)])
+ target = pa.decimal128(precision, scale)
+ actual = cast_array_for_schema_evolution(source, target)
+ assert actual.type == target
+ assert actual.to_pylist() == [None if value is None else Decimal(value)
for value in expected]
+
+
[email protected]('text', [
+ '', ' 1.2 ', '1_2', '1.2.3', '1e', 'Infinity', '0e-2147483648',
+ '0.1e-2147483647', '1e2147483647', '1e-2147483647',
'1e9999999999999999999',
+])
+def test_decimal_evolution_rejects_invalid_text(text):
+ with pytest.raises(pa.ArrowInvalid):
+ cast_array_for_schema_evolution(pa.array([None, text]),
pa.decimal128(5, 2))
+
+
+def test_decimal_evolution_handles_long_mantissas_and_java_zero_exponents():
+ values = ['1.005' + '0' * 200, '0.' + '0' * 200 + '1',
+ '0e2147483648', '0.0e2147483648', '0e2147483647']
+ result = cast_array_for_schema_evolution(pa.array(values),
pa.decimal128(5, 2))
+ assert result.to_pylist() == [Decimal('1.01')] + [Decimal('0.00')] * 4
+
+
[email protected]('source_type,values', [
+ (pa.float64(), [1.005, 10000, None]),
+ (pa.string(), ['1.005', '10000', None]),
+ (pa.decimal128(10, 3), [Decimal('1.005'), Decimal('10000'), None]),
+])
+def
test_decimal_evolution_is_independent_of_caller_decimal_context(source_type,
values):
+ source = pa.array(values, type=source_type)
+ with localcontext() as context:
+ context.prec = 2
+ context.Emax = 2
+ context.traps[Inexact] = True
+ context.traps[Rounded] = True
+ result = cast_array_for_schema_evolution(source, pa.decimal128(10, 2))
+ assert result.to_pylist() == [Decimal('1.01'), Decimal('10000.00'), None]
+
+
+def
test_decimal_evolution_is_independent_of_default_decimal_context(monkeypatch):
+ monkeypatch.setattr(DefaultContext, 'Emax', 2)
+ monkeypatch.setattr(DefaultContext, 'Emin', -2)
+ monkeypatch.setattr(DefaultContext, 'clamp', 1)
+ result = cast_array_for_schema_evolution(pa.array(['1.005', '10000',
None]), pa.decimal128(10, 2))
+ assert result.to_pylist() == [Decimal('1.01'), Decimal('10000.00'), None]
+
+
@pytest.fixture
def local_zone():
if not hasattr(time, 'tzset'):
diff --git a/paimon-python/pypaimon/utils/file_store_path_factory.py
b/paimon-python/pypaimon/utils/file_store_path_factory.py
index 8da0b11a72..c1c956e560 100644
--- a/paimon-python/pypaimon/utils/file_store_path_factory.py
+++ b/paimon-python/pypaimon/utils/file_store_path_factory.py
@@ -15,12 +15,10 @@
# specific language governing permissions and limitations
# under the License.
-import math
-import struct
from datetime import date, timezone
-from decimal import Decimal
from typing import List, Optional, Tuple
+from pypaimon.casting.numeric_utils import java_floating_string
from pypaimon.casting.row_to_string import cast_value_to_string,
_format_timestamp, _is_unsupported
from pypaimon.common.external_path_provider import ExternalPathProvider
from pypaimon.utils.path import resolve_path, to_file_io_path
@@ -82,41 +80,6 @@ def canonical_data_file_path(table, partition, bucket,
file_name):
return path
-def _floating_partition_string(value, single_precision: bool) -> str:
- # Java's JDK 19+ contract selects a shortest round-tripping form with at
- # least two significant digits before applying its notation thresholds.
- encoding = '>f' if single_precision else '>d'
- bits = struct.pack(encoding, value)
- value = struct.unpack(encoding, bits)[0]
- if math.isnan(value):
- return 'NaN'
- if math.isinf(value):
- return '-Infinity' if value < 0 else 'Infinity'
- for precision in range(2, 10 if single_precision else 18):
- text = format(value, '.{}g'.format(precision))
- try:
- rounded = struct.pack(encoding, float(text))
- except OverflowError:
- continue
- if rounded != bits:
- continue
- decimal = Decimal(text)
- if decimal.is_zero() or Decimal('0.001') <= abs(decimal) <
Decimal('1e7'):
- text = format(decimal, 'f')
- if '.' in text:
- text = text.rstrip('0').rstrip('.')
- if '.' not in text:
- text += '.0'
- else:
- mantissa, exponent = format(decimal, 'e').split('e')
- mantissa = mantissa.rstrip('0').rstrip('.') if '.' in mantissa
else mantissa
- if '.' not in mantissa:
- mantissa += '.0'
- text = '{}E{}'.format(mantissa, int(exponent))
- return text
- return str(value)
-
-
class FileStorePathFactory:
MANIFEST_PATH = "manifest"
MANIFEST_PREFIX = "manifest-"
@@ -211,7 +174,7 @@ class FileStorePathFactory:
if _is_null_or_whitespace_only(value):
text = self.default_part_value
elif type_name in ('FLOAT', 'REAL', 'DOUBLE'):
- text = _floating_partition_string(value, type_name != 'DOUBLE')
+ text = java_floating_string(value, type_name != 'DOUBLE')
elif self.legacy_partition_name and type_name == 'DATE':
text = str((value - date(1970, 1, 1)).days)
elif self.legacy_partition_name and
type_name.startswith('TIMESTAMP'):
