DRILL-6345: DRILL Query fails on Function LOG10 - Added log10 function implementation
closes #1230 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/087b366f Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/087b366f Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/087b366f Branch: refs/heads/master Commit: 087b366f211f12e274fd2967767a7fb60e836774 Parents: dbff164 Author: vladimir tkach <[email protected]> Authored: Fri Apr 27 13:24:56 2018 +0300 Committer: Vitalii Diravka <[email protected]> Committed: Sun Apr 29 23:20:55 2018 +0300 ---------------------------------------------------------------------- .../src/main/codegen/data/MathFunc.tdd | 14 ++ .../drill/exec/resolver/TypeCastRules.java | 28 +++ .../exec/fn/impl/TestNewMathFunctions.java | 238 ++++++++++++++++++- .../functions/testExtendedMathFunctions.json | 1 + 4 files changed, 269 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/087b366f/exec/java-exec/src/main/codegen/data/MathFunc.tdd ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/data/MathFunc.tdd b/exec/java-exec/src/main/codegen/data/MathFunc.tdd index feda49f..cd44916 100644 --- a/exec/java-exec/src/main/codegen/data/MathFunc.tdd +++ b/exec/java-exec/src/main/codegen/data/MathFunc.tdd @@ -254,6 +254,20 @@ extendedUnaryMathFunctions : [ ] }, + {className: "Log10", funcName: "log10", javaFunc : "java.lang.Math.log10", outputType: "Float8", types: [ + {input: "Int"}, + {input: "BigInt"}, + {input: "Float4"}, + {input: "Float8"}, + {input: "SmallInt"}, + {input: "TinyInt"}, + {input: "UInt1"}, + {input: "UInt2"}, + {input: "UInt4"}, + {input: "UInt8"} + ] + } + {className: "Degrees", funcName: "degrees", javaFunc : "java.lang.Math.toDegrees", outputType: "Float8", types: [ {input: "Int"}, {input: "BigInt"}, http://git-wip-us.apache.org/repos/asf/drill/blob/087b366f/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java index b4cf61e..0105ba0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java @@ -163,6 +163,34 @@ public class TypeCastRules { rule.add(MinorType.VARBINARY); rules.put(MinorType.BIGINT, rule); + /** UINT4 cast able from **/ + rule = new HashSet<>(); + rule.add(MinorType.TINYINT); + rule.add(MinorType.SMALLINT); + rule.add(MinorType.INT); + rule.add(MinorType.BIGINT); + rule.add(MinorType.UINT1); + rule.add(MinorType.UINT2); + rule.add(MinorType.UINT4); + rule.add(MinorType.UINT8); + rule.add(MinorType.DECIMAL9); + rule.add(MinorType.DECIMAL18); + rule.add(MinorType.DECIMAL28SPARSE); + rule.add(MinorType.DECIMAL28DENSE); + rule.add(MinorType.DECIMAL38SPARSE); + rule.add(MinorType.DECIMAL38DENSE); + rule.add(MinorType.MONEY); + rule.add(MinorType.FLOAT4); + rule.add(MinorType.FLOAT8); + rule.add(MinorType.BIT); + rule.add(MinorType.FIXEDCHAR); + rule.add(MinorType.FIXED16CHAR); + rule.add(MinorType.FIXEDBINARY); + rule.add(MinorType.VARCHAR); + rule.add(MinorType.VAR16CHAR); + rule.add(MinorType.VARBINARY); + rules.put(MinorType.UINT4, rule); + /** UINT8 cast able from **/ rule = new HashSet<>(); rule.add(MinorType.TINYINT); http://git-wip-us.apache.org/repos/asf/drill/blob/087b366f/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java index 5fd1e25..0e66b35 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java @@ -17,15 +17,10 @@ */ package org.apache.drill.exec.fn.impl; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.math.BigDecimal; - import org.apache.drill.categories.OperatorTest; import org.apache.drill.categories.UnlikelyTest; import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.exec.ExecTest; +import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.ops.FragmentContextImpl; import org.apache.drill.exec.physical.PhysicalPlan; @@ -39,16 +34,34 @@ import org.apache.drill.exec.rpc.UserClientConnection; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.VarCharVector; +import org.apache.drill.test.BaseTestQuery; + +import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.column.ParquetProperties; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.GroupWriteSupport; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; +import org.apache.commons.io.FileUtils; + import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; import com.google.common.base.Charsets; import com.google.common.io.Resources; -import org.junit.experimental.categories.Category; -import org.mockito.Mockito; +import java.io.File; +import java.math.BigDecimal; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; @Category({UnlikelyTest.class, OperatorTest.class}) -public class TestNewMathFunctions extends ExecTest { +public class TestNewMathFunctions extends BaseTestQuery { private final DrillConfig c = DrillConfig.create(); private PhysicalPlanReader reader; private FunctionImplementationRegistry registry; @@ -90,7 +103,6 @@ public class TestNewMathFunctions extends ExecTest { final PhysicalPlan plan = reader.readPhysicalPlan(planString); final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); - exec.next(); // skip schema batch while (exec.next()) { final Object [] res = getRunResult(exec); assertEquals("return count does not match", expectedResults.length, res.length); @@ -116,8 +128,21 @@ public class TestNewMathFunctions extends ExecTest { @Test public void testExtendedMathFunc() throws Throwable { final BigDecimal d = new BigDecimal("100111111111111111111111111111111111.00000000000000000000000000000000000000000000000000001"); - final Object [] expected = new Object[] {Math.cbrt(1000), Math.log(10), (Math.log(64.0)/Math.log(2.0)), Math.exp(10), Math.toDegrees(0.5), Math.toRadians(45.0), Math.PI, Math.cbrt(d.doubleValue()), Math.log(d.doubleValue()), (Math.log(d.doubleValue())/Math.log(2)), Math.exp(d.doubleValue()), Math.toDegrees(d.doubleValue()), Math.toRadians(d.doubleValue())}; - + final Object [] expected = new Object[] { + Math.cbrt(1000), + Math.log(10), + Math.log10(5), + (Math.log(64.0) / Math.log(2.0)), + Math.exp(10), Math.toDegrees(0.5), + Math.toRadians(45.0), + Math.PI, + Math.cbrt(d.doubleValue()), + Math.log(d.doubleValue()), + (Math.log(d.doubleValue()) / Math.log(2)), + Math.exp(d.doubleValue()), + Math.toDegrees(d.doubleValue()), + Math.toRadians(d.doubleValue()) + }; runTest(expected, "functions/testExtendedMathFunctions.json"); } @@ -132,4 +157,193 @@ public class TestNewMathFunctions extends ExecTest { final Object [] expected = new Object[] {1, 1, 1, 0}; runTest(expected, "functions/testIsNumericFunction.json"); } + + @Test + public void testLog10WithDouble() throws Throwable { + String json = "{" + + "\"positive_infinity\" : Infinity," + + "\"negative_infinity\" : -Infinity," + + "\"nan\" : NaN," + + "\"num1\": 0.0," + + "\"num2\": 0.1," + + "\"num3\": 1.0," + + "\"num4\": 1.5," + + "\"num5\": -1.5," + + "\"num6\": 10.0" + + "}"; + String query = + "select " + + "log10(positive_infinity) as pos_inf, " + + "log10(negative_infinity) as neg_inf, " + + "log10(nan) as nan, " + + "log10(num1) as num1, " + + "log10(num2) as num2, " + + "log10(num3) as num3, " + + "log10(num4) as num4, " + + "log10(num5) as num5, " + + "log10(num6) as num6 " + + "from dfs.`data.json`"; + File file = new File(dirTestWatcher.getRootDir(), "data.json"); + try { + FileUtils.writeStringToFile(file, json); + setSessionOption(ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE, true); + setSessionOption(ExecConstants.JSON_READER_NAN_INF_NUMBERS, true); + testBuilder() + .sqlQuery(query) + .ordered() + .baselineColumns("pos_inf", "neg_inf", "nan", "num1", "num2", "num3", "num4", "num5", "num6") + .baselineValues(Double.POSITIVE_INFINITY, Double.NaN, Double.NaN, Double.NEGATIVE_INFINITY, + -1.0d, 0d, 0.17609125905568124d, Double.NaN, 1.0d) + .go(); + } finally { + resetSessionOption(ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE); + resetSessionOption(ExecConstants.JSON_READER_NAN_INF_NUMBERS); + FileUtils.deleteQuietly(file); + } + } + + @Test + public void testLog10WithFloat() throws Throwable { + String json = "{" + + "\"positive_infinity\" : Infinity," + + "\"negative_infinity\" : -Infinity," + + "\"nan\" : NaN," + + "\"num1\": 0.0," + + "\"num2\": 0.1," + + "\"num3\": 1.0," + + "\"num4\": 1.5," + + "\"num5\": -1.5," + + "\"num6\": 10.0" + + "}"; + String query = + "select " + + "log10(cast(positive_infinity as float)) as pos_inf, " + + "log10(cast(negative_infinity as float)) as neg_inf, " + + "log10(cast(nan as float)) as nan, " + + "log10(cast(num1 as float)) as num1, " + + "log10(cast(num2 as float)) as num2, " + + "log10(cast(num3 as float)) as num3, " + + "log10(cast(num4 as float)) as num4, " + + "log10(cast(num5 as float)) as num5, " + + "log10(cast(num6 as float)) as num6 " + + "from dfs.`data.json`"; + File file = new File(dirTestWatcher.getRootDir(), "data.json"); + try { + FileUtils.writeStringToFile(file, json); + setSessionOption(ExecConstants.JSON_READER_NAN_INF_NUMBERS, true); + testBuilder() + .sqlQuery(query) + .ordered() + .baselineColumns("pos_inf", "neg_inf", "nan", "num1", "num2", "num3", "num4", "num5", "num6") + .baselineValues(Double.POSITIVE_INFINITY, Double.NaN, Double.NaN, Double.NEGATIVE_INFINITY, + -0.999999993528508d, 0d, 0.17609125905568124d, Double.NaN, 1.0d) + .go(); + } finally { + resetSessionOption(ExecConstants.JSON_READER_NAN_INF_NUMBERS); + FileUtils.deleteQuietly(file); + } + } + + @Test + public void testLog10WithInt() throws Throwable { + String json = "{" + + "\"num1\": 0.0," + + "\"num3\": 1.0," + + "\"num5\": -1.0," + + "\"num6\": 10.0" + + "}"; + String query = + "select " + + "log10(cast(num1 as int)) as num1, " + + "log10(cast(num3 as int)) as num3, " + + "log10(cast(num5 as int)) as num5, " + + "log10(cast(num6 as int)) as num6 " + + "from dfs.`data.json`"; + File file = new File(dirTestWatcher.getRootDir(), "data.json"); + try { + FileUtils.writeStringToFile(file, json); + setSessionOption(ExecConstants.JSON_READER_NAN_INF_NUMBERS, true); + testBuilder() + .sqlQuery(query) + .ordered() + .baselineColumns("num1", "num3", "num5", "num6") + .baselineValues(Double.NEGATIVE_INFINITY, 0d, Double.NaN, 1.0d) + .go(); + } finally { + resetSessionOption(ExecConstants.JSON_READER_NAN_INF_NUMBERS); + FileUtils.deleteQuietly(file); + } + } + + @Test + public void testLog10WithBigInt() throws Throwable { + String json = "{" + + "\"num1\": 0," + + "\"num3\": 1," + + "\"num5\": -1," + + "\"num6\": 10" + + "}"; + String query = + "select " + + "log10(num1) as num1, " + + "log10(num3) as num3, " + + "log10(num5) as num5, " + + "log10(num6) as num6 " + + "from dfs.`data.json`"; + File file = new File(dirTestWatcher.getRootDir(), "data.json"); + try { + FileUtils.writeStringToFile(file, json); + + setSessionOption(ExecConstants.JSON_READER_NAN_INF_NUMBERS, true); + testBuilder() + .sqlQuery(query) + .ordered() + .baselineColumns("num1", "num3", "num5", "num6") + .baselineValues(Double.NEGATIVE_INFINITY, 0d, Double.NaN, 1.0d) + .go(); + } finally { + resetSessionOption(ExecConstants.JSON_READER_NAN_INF_NUMBERS); + FileUtils.deleteQuietly(file); + } + } + + + @Test + public void testLog10WithUint4() throws Exception { + org.apache.hadoop.fs.Path file = new org.apache.hadoop.fs.Path(dirTestWatcher.getRootDir().toString(), "uint8.parquet"); + String schemaText = "message test { required int32 val(UINT_8); }"; + + Configuration conf = new Configuration(); + MessageType schema = MessageTypeParser.parseMessageType(schemaText); + GroupWriteSupport.setSchema(schema, conf); + SimpleGroupFactory groupFactory = new SimpleGroupFactory(schema); + + try { + try (ParquetWriter<Group> writer = new ParquetWriter<>( + file, + new GroupWriteSupport(), + CompressionCodecName.UNCOMPRESSED, 1024, 1024, 512, true, false, + ParquetProperties.WriterVersion.PARQUET_1_0, conf)) { + + writer.write(groupFactory.newGroup().append("val", 0)); + writer.write(groupFactory.newGroup().append("val", 1)); + writer.write(groupFactory.newGroup().append("val", -1)); + writer.write(groupFactory.newGroup().append("val", 10)); + } + String query = "select log10(val) as col from dfs.`uint8.parquet`"; + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("col") + .baselineValues(Double.NEGATIVE_INFINITY) + .baselineValues(0d) + .baselineValues(Double.NaN) + .baselineValues(1.0d) + .build() + .run(); + } finally { + FileUtils.deleteQuietly(new File(file.toString())); + } + } + } http://git-wip-us.apache.org/repos/asf/drill/blob/087b366f/exec/java-exec/src/test/resources/functions/testExtendedMathFunctions.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/functions/testExtendedMathFunctions.json b/exec/java-exec/src/test/resources/functions/testExtendedMathFunctions.json index 9814cf6..86570dc 100644 --- a/exec/java-exec/src/test/resources/functions/testExtendedMathFunctions.json +++ b/exec/java-exec/src/test/resources/functions/testExtendedMathFunctions.json @@ -23,6 +23,7 @@ exprs : [ { ref : "CBRTVAL", expr : " cbrt(1000) "}, { ref : "LOGVAL", expr : " log(10) "}, + { ref : "LOG10VAL", expr : " log10(5) "}, { ref : "LOGVALBASE", expr : " log(2.0, 64.0) "}, { ref : "EXP", expr : " exp(10) "}, { ref : "DEGREES", expr : " degrees(0.5) "},
