(calcite) branch main updated: [CALCITE-6533] Division between integer and decimal produces incorrect result

[mbudiu]

This is an automated email from the ASF dual-hosted git repository.

mbudiu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git


The following commit(s) were added to refs/heads/main by this push:
     new ac06dbcf50 [CALCITE-6533] Division between integer and decimal 
produces incorrect result
ac06dbcf50 is described below

commit ac06dbcf5016ced9f158eb548c3348dcf62d7de7
Author: Mihai Budiu <mbu...@feldera.com>
AuthorDate: Sat Aug 17 10:59:03 2024 -0700

    [CALCITE-6533] Division between integer and decimal produces incorrect 
result
    
    Signed-off-by: Mihai Budiu <mbu...@feldera.com>
---
 .../org/apache/calcite/rel/type/RelDataTypeSystem.java    |  3 ++-
 .../java/org/apache/calcite/test/SqlOperatorTest.java     | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git 
a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystem.java 
b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystem.java
index ccc9537d6f..ba2b0e6f60 100644
--- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystem.java
+++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystem.java
@@ -297,6 +297,7 @@ public interface RelDataTypeSystem {
         int six = Math.min(6, getMaxNumericScale());
         int d = p1 - s1 + s2;
         int scale = Math.max(six, s1 + p2 + 1);
+        scale = Math.min(scale, getMaxNumericScale());
         int precision = d + scale;
 
   // Rules from
@@ -320,7 +321,7 @@ public interface RelDataTypeSystem {
         //   can't fit into 32 digits.
         int bound = getMaxNumericPrecision() - six;  // This was '32' in the 
MS documentation
         if (precision <= bound) {
-          scale = Math.min(scale, getMaxNumericScale() - (precision - scale));
+          scale = Math.min(scale, getMaxNumericPrecision() - (precision - 
scale));
         } else {
           // precision > bound
           scale = Math.min(six, scale);
diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java 
b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
index d6563f3aff..1fa79f3099 100644
--- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
+++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
@@ -2643,6 +2643,8 @@ public class SqlOperatorTest {
   @Test void testDivideOperator() {
     final SqlOperatorFixture f = fixture();
     f.setFor(SqlStdOperatorTable.DIVIDE, VmName.EXPAND);
+    f.checkScalarExact("95.0 / 100", "DECIMAL(14, 6) NOT NULL", "0.95");
+    f.checkScalarExact("95 / 100.0", "DECIMAL(17, 6) NOT NULL", "0.95");
     f.checkScalarExact("10 / 5", "INTEGER NOT NULL", "2");
     f.checkScalarExact("-10 / 5", "INTEGER NOT NULL", "-2");
     f.checkScalarExact("-10 / 5.0", "DECIMAL(17, 6) NOT NULL", "-2");
@@ -2661,6 +2663,19 @@ public class SqlOperatorTest {
     f.checkNull("1e1 / cast(null as float)");
     f.checkScalarExact("100.1 / 0.00000000000000001", "DECIMAL(19, 6) NOT 
NULL",
         "1.001E+19");
+    SqlOperatorFixture f0 = f.withFactory(tf ->
+        tf.withTypeSystem(typeSystem ->
+            new DelegatingTypeSystem(typeSystem) {
+              @Override public int getMaxNumericPrecision() {
+                return 28;
+              }
+
+              @Override public int getMaxNumericScale() {
+                return 10;
+              }
+            }));
+    f0.checkScalarExact("95.0 / 100", "DECIMAL(12, 10) NOT NULL", "0.95");
+    f0.checkScalarExact("95 / 100.0", "DECIMAL(17, 6) NOT NULL", "0.95");
   }
 
   @Test void testDivideOperatorIntervals() {