This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 6b21940693 Issue #6111 : Avoid scientific notation when converting
Number to BigNumber (#7744)
6b21940693 is described below
commit 6b21940693d1a17db2fc5b368ff5aba810f67b0b
Author: vbhanuchander-lang <[email protected]>
AuthorDate: Mon Aug 3 02:57:27 2026 -0400
Issue #6111 : Avoid scientific notation when converting Number to BigNumber
(#7744)
BigDecimal.valueOf(double) is specified as new
BigDecimal(Double.toString(val)),
and Double.toString() switches to scientific notation from 10^7 onwards. For
55487400.0 it returns "5.54874E7", which parses into an unscaled value of
554874
with a scale of -2. It is that negative scale which makes
BigDecimal.toString()
render the value as "5.54874E+7" again, so every consumer that serializes a
BigNumber through toString() emits scientific notation for a value that
started
out as plain digits:
- JDBC drivers that inline statement parameters, which is how the Table
Output
INSERT in the reported case ends up carrying 5.54874E+7
- writeBigNumber(), used by the binary row serialization
- getDataXml(), used by the XML data serialization
Route the four Double to BigDecimal conversions in ValueMetaBase through a
helper
that rescales to zero when the scale would be negative. Rescaling upwards
from a
negative scale is exact, so no rounding takes place and the numeric value is
unchanged. Values that already convert to a non-negative scale keep the
exact
BigDecimal they produced before.
The string conversion paths format through DecimalFormat and never emitted
scientific notation; a regression test pins that behaviour down as well.
---
.../apache/hop/core/row/value/ValueMetaBase.java | 30 +++++-
.../hop/core/row/value/ValueMetaBaseTest.java | 106 +++++++++++++++++++++
2 files changed, 132 insertions(+), 4 deletions(-)
diff --git
a/core/src/main/java/org/apache/hop/core/row/value/ValueMetaBase.java
b/core/src/main/java/org/apache/hop/core/row/value/ValueMetaBase.java
index 5dae65ca35..823d60ca34 100644
--- a/core/src/main/java/org/apache/hop/core/row/value/ValueMetaBase.java
+++ b/core/src/main/java/org/apache/hop/core/row/value/ValueMetaBase.java
@@ -1624,6 +1624,28 @@ public class ValueMetaBase implements IValueMeta {
}
}
+ /**
+ * Converts a double into a BigDecimal that does not carry a negative scale.
+ *
+ * <p>{@link BigDecimal#valueOf(double)} is built from {@link
Double#toString(double)}, which
+ * switches to scientific notation from 10<sup>7</sup> onwards. The
resulting string "5.54874E7"
+ * parses into an unscaled value of 554874 with a scale of -2, and it is
that negative scale which
+ * makes {@link BigDecimal#toString()} render the value as "5.54874E+7"
again. Every consumer that
+ * serializes the value through toString() - JDBC drivers that inline
statement parameters, {@link
+ * #writeBigNumber(java.io.DataOutputStream, BigDecimal)} and {@link
#getDataXml(Object)} - then
+ * emits scientific notation for a value that started out as plain digits.
+ *
+ * <p>Rescaling to zero is exact whenever the scale is negative, so the
numeric value is left
+ * untouched and no rounding can occur.
+ *
+ * @param number the double to convert
+ * @return the value as a BigDecimal, rescaled to zero when the scale would
be negative
+ */
+ protected static BigDecimal convertDoubleToBigNumber(double number) {
+ BigDecimal bigDecimal = BigDecimal.valueOf(number);
+ return bigDecimal.scale() < 0 ? bigDecimal.setScale(0) : bigDecimal;
+ }
+
protected synchronized BigDecimal convertStringToBigNumber(String string)
throws HopValueException {
string = Const.trimToType(string, getTrimType()); // see if trimming needs
@@ -1658,7 +1680,7 @@ public class ValueMetaBase implements IValueMeta {
// If the Number is not a BigDecimal.
//
if (number instanceof Double) {
- return BigDecimal.valueOf(number.doubleValue());
+ return convertDoubleToBigNumber(number.doubleValue());
} else if (number instanceof Long) {
return BigDecimal.valueOf(number.longValue());
}
@@ -2605,11 +2627,11 @@ public class ValueMetaBase implements IValueMeta {
};
case TYPE_NUMBER:
return switch (storageType) {
- case STORAGE_TYPE_NORMAL -> BigDecimal.valueOf((Double) object);
+ case STORAGE_TYPE_NORMAL -> convertDoubleToBigNumber((Double)
object);
case STORAGE_TYPE_BINARY_STRING ->
- BigDecimal.valueOf((Double)
convertBinaryStringToNativeType((byte[]) object));
+ convertDoubleToBigNumber((Double)
convertBinaryStringToNativeType((byte[]) object));
case STORAGE_TYPE_INDEXED ->
- BigDecimal.valueOf((Double) index[((Integer)
object).intValue()]);
+ convertDoubleToBigNumber((Double) index[((Integer)
object).intValue()]);
default ->
throw new HopValueException(
this
diff --git
a/core/src/test/java/org/apache/hop/core/row/value/ValueMetaBaseTest.java
b/core/src/test/java/org/apache/hop/core/row/value/ValueMetaBaseTest.java
index 1d09e760cb..a5bc242c96 100644
--- a/core/src/test/java/org/apache/hop/core/row/value/ValueMetaBaseTest.java
+++ b/core/src/test/java/org/apache/hop/core/row/value/ValueMetaBaseTest.java
@@ -443,6 +443,112 @@ class ValueMetaBaseTest {
assertThrows(HopValueException.class, () ->
valueMeta.getBigNumber("1234567890"));
}
+ @Test
+ void testGetBigNumberFromNumberKeepsPlainNotation() throws HopValueException
{
+ // BigDecimal.valueOf(55487400.0) is built from Double.toString(), which
returns "5.54874E7".
+ // The resulting BigDecimal has an unscaled value of 554874 and a scale of
-2, so rendering it
+ // with toString() yields "5.54874E+7" rather than "55487400".
+ ValueMetaBase valueMeta = new ValueMetaNumber("float53");
+
+ BigDecimal bigNumber = valueMeta.getBigNumber(55487400.0d);
+
+ assertEquals(0, bigNumber.compareTo(new BigDecimal("55487400")));
+ assertTrue(bigNumber.scale() >= 0, "unexpected negative scale: " +
bigNumber.scale());
+ assertEquals("55487400", bigNumber.toString());
+ }
+
+ @Test
+ void testGetBigNumberFromNumberKeepsPlainNotationAcrossMagnitudes() throws
HopValueException {
+ ValueMetaBase valueMeta = new ValueMetaNumber("float53");
+ double[] values = {1.0e7, 5.54874e7, 1.23456789e8, 1.5e10,
9.007199254740992e15, -5.54874e7};
+
+ for (double value : values) {
+ BigDecimal bigNumber = valueMeta.getBigNumber(value);
+
+ assertTrue(bigNumber.scale() >= 0, "negative scale for " + value + ": "
+ bigNumber.scale());
+ assertEquals(
+ bigNumber.toPlainString(), bigNumber.toString(), "scientific
notation for " + value);
+ assertEquals(0, bigNumber.compareTo(BigDecimal.valueOf(value)), "value
changed for " + value);
+ }
+ }
+
+ @Test
+ void testGetBigNumberFromNumberLeavesNonNegativeScalesUntouched() throws
HopValueException {
+ // Values that already convert to a non-negative scale must keep the exact
BigDecimal they
+ // produced before, scale included. BigDecimal.equals() is scale
sensitive, so this pins that
+ // down rather than only comparing numeric values.
+ ValueMetaBase valueMeta = new ValueMetaNumber("number");
+ double[] values = {0.0, 0.1, 123.45, 1.0e-7, 55487400.5};
+
+ for (double value : values) {
+ assertEquals(BigDecimal.valueOf(value), valueMeta.getBigNumber(value),
"changed " + value);
+ }
+ }
+
+ @Test
+ void testConvertNumberToBigNumberKeepsPlainNotation() throws
HopValueException {
+ // The metadata change a Select Values transform performs when a Number
field is turned into a
+ // BigNumber field.
+ IValueMeta source = new ValueMetaNumber("float53");
+ IValueMeta target = new ValueMetaBigNumber("decimal");
+
+ BigDecimal converted = (BigDecimal) target.convertData(source,
55487400.0d);
+
+ assertEquals("55487400", converted.toString());
+ }
+
+ @Test
+ void testGetBigNumberFromStringKeepsPlainNotation() throws HopValueException
{
+ ValueMetaBase valueMeta = new ValueMetaString("float53");
+
+ BigDecimal bigNumber = valueMeta.getBigNumber("55487400");
+
+ assertTrue(bigNumber.scale() >= 0, "unexpected negative scale: " +
bigNumber.scale());
+ assertEquals(0, bigNumber.compareTo(new BigDecimal("55487400")));
+ }
+
+ @Test
+ void testWriteBigNumberConvertedFromNumberUsesPlainNotation() throws
Exception {
+ // writeBigNumber() serializes with BigDecimal.toString(), so a negative
scale would put
+ // scientific notation on the wire as well.
+ BigDecimal bigNumber = new
ValueMetaNumber("float53").getBigNumber(55487400.0d);
+ ValueMetaBase valueMeta = new ValueMetaBigNumber("decimal");
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (DataOutputStream outputStream = new DataOutputStream(out)) {
+ valueMeta.writeData(outputStream, bigNumber);
+ }
+
+ BigDecimal restored;
+ try (DataInputStream inputStream =
+ new DataInputStream(new ByteArrayInputStream(out.toByteArray()))) {
+ restored = (BigDecimal) valueMeta.readData(inputStream);
+ }
+
+ // readBigNumber() keeps whatever scale was written, so a plain result
proves a plain wire form.
+ assertEquals("55487400", restored.toString());
+ }
+
+ @Test
+ void testGetDataXmlForBigNumberConvertedFromNumberUsesPlainNotation() throws
Exception {
+ BigDecimal bigNumber = new
ValueMetaNumber("float53").getBigNumber(55487400.0d);
+ ValueMetaBase valueMeta = new ValueMetaBigNumber("decimal");
+
+ String xml = valueMeta.getDataXml(bigNumber);
+
+ assertTrue(xml.contains("55487400"), xml);
+ assertFalse(xml.contains("E+"), xml);
+ }
+
+ @Test
+ void testConvertNumberToStringDoesNotUseScientificNotation() throws
HopValueException {
+ // Guards the string conversion path, which formats through DecimalFormat
and was already
+ // correct, against a regression from the BigDecimal change.
+ assertFalse(new
ValueMetaNumber("float53").getString(55487400.0d).contains("E"));
+ assertFalse(
+ new ValueMetaBigNumber("decimal").getString(new
BigDecimal("5.54874E+7")).contains("E"));
+ }
+
@Test
void testGetIntegerThrowsHopValueException() {
ValueMetaBase valueMeta = new ValueMetaInteger();
