[Libreoffice-commits] core.git: sal/rtl sc/qa

2020-12-02 Thread Eike Rathke (via logerrit) 
 sal/rtl/math.cxx  |   24 
++
 sc/qa/unit/data/functions/addin/fods/convert.fods |6 +-
 sc/qa/unit/data/functions/financial/fods/nper.fods|2 
 sc/qa/unit/data/functions/fods/gammaln.precise.fods   |4 -
 sc/qa/unit/data/functions/mathematical/fods/convert_add.fods  |6 +-
 sc/qa/unit/data/functions/statistical/fods/chisq.inv.fods |   12 ++---
 sc/qa/unit/data/functions/statistical/fods/forecast.ets.mult.fods |   14 ++---
 sc/qa/unit/data/functions/statistical/fods/gammaln.fods   |4 -
 sc/qa/unit/data/functions/statistical/fods/gammaln.precise.fods   |4 -
 sc/qa/unit/data/functions/statistical/fods/geomean.fods   |2 
 sc/qa/unit/data/functions/statistical/fods/harmean.fods   |6 +-
 sc/qa/unit/data/functions/statistical/fods/lognorm.inv.fods   |4 -
 sc/qa/unit/data/functions/statistical/fods/stdev.fods |4 -
 sc/qa/unit/data/functions/statistical/fods/stdev.p.fods   |6 +-
 sc/qa/unit/data/functions/statistical/fods/stdev.s.fods   |4 -
 sc/qa/unit/data/functions/statistical/fods/stdeva.fods|4 -
 sc/qa/unit/data/functions/statistical/fods/stdevp.fods|6 +-
 17 files changed, 49 insertions(+), 63 deletions(-)

New commits:
commit deb119e415213716a76b9b489a700949c031c6fe
Author: Eike Rathke 
AuthorDate: Wed Dec 2 22:21:12 2020 +0100
Commit: Eike Rathke 
CommitDate: Thu Dec 3 01:28:18 2020 +0100

Related: tdf#138360 Use approxFloor() in rtl_math_round()

Ditch mostly but not always working correction value and use
approxFloor() instead which yields better results. With this we
now have one single place approxValue() in case more fine grained
tuning is needed.

Unfortunately all numeric spreadsheet function tests use ROUND()
in a manner such that they mostly blindly do a ROUND(...;12)
regardless of magnitude, sometimes effectively rounding to the
14th significant digit that may fail in cases like for

14.2040730851385
  ^
where the constant (rounded) value is stored as is but the
calculated value is
14.204073085138471
and the old round() yielded
14.204073085139 for both but the new round() more correctly
results in
14.204073085139 and
14.204073085138
so the spreadsheet test case sources had to be changed to
ROUND(...;11) in such places.

Change-Id: I211bb868a4f4dc9e68f4c7dcc2a187b5e175416f
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/107135
Reviewed-by: Eike Rathke 
Tested-by: Jenkins

diff --git a/sal/rtl/math.cxx b/sal/rtl/math.cxx
index e6f09f18030e..a296927635bf 100644
--- a/sal/rtl/math.cxx
+++ b/sal/rtl/math.cxx
@@ -1133,6 +1133,9 @@ double SAL_CALL rtl_math_round(double fValue, int 
nDecPlaces,
 {
 OSL_ASSERT(nDecPlaces >= -20 && nDecPlaces <= 20);
 
+if (!std::isfinite(fValue))
+return fValue;
+
 if (fValue == 0.0)
 return fValue;
 
@@ -1190,24 +1193,7 @@ double SAL_CALL rtl_math_round(double fValue, int 
nDecPlaces,
 switch ( eMode )
 {
 case rtl_math_RoundingMode_Corrected :
-{
-int nExp;   // exponent for correction
-if ( fValue > 0.0 )
-nExp = static_cast( floor( log10( fValue ) ) );
-else
-nExp = 0;
-
-int nIndex;
-
-if (nExp < 0)
-nIndex = 15;
-else if (nExp >= 14)
-nIndex = 0;
-else
-nIndex = 15 - nExp;
-
-fValue = floor(fValue + 0.5 + nCorrVal[nIndex]);
-}
+fValue = rtl::math::approxFloor(fValue + 0.5);
 break;
 case rtl_math_RoundingMode_Down:
 fValue = rtl::math::approxFloor(fValue);
@@ -1321,7 +1307,7 @@ double SAL_CALL rtl_math_approxValue( double fValue ) 
SAL_THROW_EXTERN_C()
 if (!std::isfinite(fValue))
 return fOrigValue;
 
-fValue = rtl_math_round(fValue, 0, rtl_math_RoundingMode_Corrected);
+fValue = std::round(fValue);
 fValue /= fExpValue;
 
 // If the original value was near DBL_MAX we got an overflow. Restore and
diff --git a/sc/qa/unit/data/functions/addin/fods/convert.fods 
b/sc/qa/unit/data/functions/addin/fods/convert.fods
index 12ba09dcd326..64eb2db5ff82 100644
--- a/sc/qa/unit/data/functions/addin/fods/convert.fods
+++ b/sc/qa/unit/data/functions/addin/fods/convert.fods
@@ -3409,7 +3409,7 @@
  
   14.2857127610345
  
- 
+ 
   TRUE
  
  
@@ -4425,7 +4425,7 @@
  
   11.1445349270435
  
- 
+ 
   TRUE
  
  
@@ -5867,4 +5867,4 @@

   
  
-
\ No newline at end of file
+
diff --git a/sc/qa/unit/data/functions/financial/fods/nper.fods 
b/sc/qa/unit/data/functions/financial/fods/nper.fods
index 2eac9b8f339f..83dc438afa34 100644
---

[Libreoffice-commits] core.git: sal/rtl sc/qa

2018-01-16 Thread Winfried Donkers 
 sal/rtl/math.cxx   |3 
 sc/qa/unit/data/functions/mathematical/fods/round.fods |   45 +++-
 sc/qa/unit/data/functions/statistical/fods/chisq.test.fods |   50 ++---
 sc/qa/unit/data/functions/statistical/fods/chisqdist.fods  |   40 ++--
 sc/qa/unit/data/functions/statistical/fods/chisqinv.fods   |  118 ++---
 sc/qa/unit/data/functions/statistical/fods/chitest.fods|   50 ++---
 sc/qa/unit/data/functions/statistical/fods/stdevpa.fods|   28 +--
 sc/qa/unit/data/functions/statistical/fods/t.dist.rt.fods  |   66 +++
 sc/qa/unit/data/functions/statistical/fods/tinv.fods   |   60 +++---
 9 files changed, 243 insertions(+), 217 deletions(-)

New commits:
commit b97a0df0f3234b4c1140ba1418d4b96a592afa4a
Author: Winfried Donkers 
Date:   Sat Dec 30 18:25:02 2017 +0100

tdf#96821 fix corner cases for Calc function ROUND.

For very large integer numbers, e.g. 2^52+1, ROUND produced incorrect
results because rtl::math::round uses
floor( number + 0.5 + small_correction_value ), which reduces the maximum
possible mantissa resolution.a

Correction of several unit test documents (Calc functions) with rounding
errors that only became apparent with this fix.

Change-Id: I1769c9939a3d6118d3bfbfdf8e41dd4619997232
Reviewed-on: https://gerrit.libreoffice.org/47179
Tested-by: Jenkins 
Reviewed-by: Eike Rathke 

diff --git a/sal/rtl/math.cxx b/sal/rtl/math.cxx
index 4b149b9f99dd..387ae3b2b4aa 100644
--- a/sal/rtl/math.cxx
+++ b/sal/rtl/math.cxx
@@ -1076,6 +1076,9 @@ double SAL_CALL rtl_math_round(double fValue, int 
nDecPlaces,
 if (fValue == 0.0)
 return fValue;
 
+if ( nDecPlaces == 0 && eMode == rtl_math_RoundingMode_Corrected )
+return std::round( fValue );
+
 // sign adjustment
 bool bSign = rtl::math::isSignBitSet( fValue );
 if (bSign)
diff --git a/sc/qa/unit/data/functions/mathematical/fods/round.fods 
b/sc/qa/unit/data/functions/mathematical/fods/round.fods
index 0f0a5b9026c3..d75968c17bbe 100644
--- a/sc/qa/unit/data/functions/mathematical/fods/round.fods
+++ b/sc/qa/unit/data/functions/mathematical/fods/round.fods
@@ -2117,20 +2117,43 @@
  
 
 
- 
- 
- 
- 
- 
+ 
+  4503599627370497
+ 
+ 
+  4503599627370497
+ 
+ 
+  TRUE
+ 
+ 
+  =ROUND(2^52+1,0)
+ 
+ 
+  limit of resolution
+ 
+ 
  
  
  
 
-
- 
- 
- 
- 
+
+ 
+  4503599627370497
+ 
+ 
+  4503599627370497
+ 
+ 
+  TRUE
+ 
+ 
+  =ROUND(2^52+1)
+ 
+ 
+  limit of resolution
+ 
+ 
  
  
 
@@ -2380,4 +2403,4 @@

   
  
-
\ No newline at end of file
+
diff --git a/sc/qa/unit/data/functions/statistical/fods/chisq.test.fods 
b/sc/qa/unit/data/functions/statistical/fods/chisq.test.fods
index 9eac30114eb7..648f8b864679 100644
--- a/sc/qa/unit/data/functions/statistical/fods/chisq.test.fods
+++ b/sc/qa/unit/data/functions/statistical/fods/chisq.test.fods
@@ -3803,7 +3803,7 @@
  
   1.16440189336067E-29
  
- 
+ 
   TRUE
  
  
@@ -3866,7 +3866,7 @@
  
   9.03490480352966E-10
  
- 
+ 
   TRUE
  
  
@@ -3934,7 +3934,7 @@
  
   1.34423459890025E-09
  
- 
+ 
   TRUE
  
  
@@ -4000,7 +4000,7 @@
  
   0
  
- 
+ 
   TRUE
  
  
@@ -4058,7 +4058,7 @@
  
   1.12896923737676E-11
  
- 
+ 
   TRUE
  
  
@@ -4121,7 +4121,7 @@
  
   0.997916658208002
  
- 
+ 
   TRUE
  
  
@@ -4184,7 +4184,7 @@
  
   0.001549713633046
  
- 
+ 
   TRUE
  
  
@@ -4246,7 +4246,7 @@
  
   0.020252473072358
  
- 
+ 
   TRUE
  
  
@@ -4306,7 +4306,7 @@
  
   1.66789802691649E-16
  
- 
+ 
   TRUE
  
  
@@ -4358,7 +4358,7 @@
  
   0
  
- 
+ 
   TRUE
  
  
@@ -4403,7 +4403,7 @@
  
   0.000308192017008
  
- 
+ 
   TRUE
  
  
@@ -4435,7 +4435,7 @@
  
   0.816527699624637
  
- 
+ 
   TRUE
  
  
@@ -4466,7 +4466,7 @@
  
   0.87928370831036
  
- 
+ 
   TRUE
  
  
@@ -4507,7 +4507,7 @@
  
   0.911826184650631
  
- 
+ 
   TRUE
  
  
@@ -4550,7 +4550,7 @@
  
   0.9314883584606
  
- 
+ 
   TRUE
  
  
@@ -4581,7 +4581,7 @@
  
   0.944539492146675
  
- 
+ 
   TRUE
  
  
@@ -4617,7 +4617,7 @@
  
   0.953774634535083
  
- 
+ 
   TRUE
  
  
@@ -4655,7 +4655,7 @@
  
   0.960619865110165
  
- 
+ 
   TRUE
  
  
@@ -4681,7 +4681,7