http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 654736d..aa3f1c2 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -23,7 +23,7 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.exception.CatalogException; import org.apache.tajo.catalog.exception.NoSuchFunctionException; -import org.apache.tajo.catalog.function.Function; +import org.apache.tajo.function.Function; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType; @@ -499,7 +499,7 @@ public class TestCatalog { assertTrue(catalog.containFunction("test10", CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.BLOB))); FunctionDesc retrived = catalog.getFunction("test10", CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.BLOB)); - assertEquals(retrived.getSignature(), "test10"); + assertEquals(retrived.getFunctionName(), "test10"); assertEquals(retrived.getFuncClass(), TestFunc2.class); assertEquals(retrived.getFuncType(), FunctionType.GENERAL); @@ -518,7 +518,7 @@ public class TestCatalog { assertTrue(catalog.containFunction("test2", CatalogUtil.newSimpleDataTypeArray(Type.INT4))); FunctionDesc retrived = catalog.getFunction("test2", CatalogUtil.newSimpleDataTypeArray(Type.INT4)); - assertEquals(retrived.getSignature(),"test2"); + assertEquals(retrived.getFunctionName(),"test2"); assertEquals(retrived.getFuncClass(),TestFunc1.class); assertEquals(retrived.getFuncType(),FunctionType.UDF); } @@ -845,7 +845,7 @@ public class TestCatalog { // UPGRADE TO INT4 SUCCESS==> LOOK AT SECOND PARAM BELOW FunctionDesc retrieved = catalog.getFunction("testint", CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.INT2)); - assertEquals(retrieved.getSignature(), "testint"); + assertEquals(retrieved.getFunctionName(), "testint"); assertEquals(retrieved.getParamTypes()[0], CatalogUtil.newSimpleDataType(Type.INT4)); assertEquals(retrieved.getParamTypes()[1] , CatalogUtil.newSimpleDataType(Type.INT4)); } @@ -874,7 +874,7 @@ public class TestCatalog { FunctionDesc retrieved = catalog.getFunction("testfloat", CatalogUtil.newSimpleDataTypeArray(Type.FLOAT4, Type.INT4)); - assertEquals(retrieved.getSignature(), "testfloat"); + assertEquals(retrieved.getFunctionName(), "testfloat"); assertEquals(retrieved.getParamTypes()[0], CatalogUtil.newSimpleDataType(Type.FLOAT8)); assertEquals(retrieved.getParamTypes()[1] , CatalogUtil.newSimpleDataType(Type.INT4)); } @@ -900,19 +900,19 @@ public class TestCatalog { assertTrue(catalog.createFunction(meta)); FunctionDesc retrieved = catalog.getFunction("testany", CatalogUtil.newSimpleDataTypeArray(Type.INT1)); - assertEquals(retrieved.getSignature(), "testany"); + assertEquals(retrieved.getFunctionName(), "testany"); assertEquals(retrieved.getParamTypes()[0], CatalogUtil.newSimpleDataType(Type.ANY)); retrieved = catalog.getFunction("testany", CatalogUtil.newSimpleDataTypeArray(Type.INT8)); - assertEquals(retrieved.getSignature(), "testany"); + assertEquals(retrieved.getFunctionName(), "testany"); assertEquals(retrieved.getParamTypes()[0], CatalogUtil.newSimpleDataType(Type.ANY)); retrieved = catalog.getFunction("testany", CatalogUtil.newSimpleDataTypeArray(Type.FLOAT4)); - assertEquals(retrieved.getSignature(), "testany"); + assertEquals(retrieved.getFunctionName(), "testany"); assertEquals(retrieved.getParamTypes()[0], CatalogUtil.newSimpleDataType(Type.ANY)); retrieved = catalog.getFunction("testany", CatalogUtil.newSimpleDataTypeArray(Type.TEXT)); - assertEquals(retrieved.getSignature(), "testany"); + assertEquals(retrieved.getFunctionName(), "testany"); assertEquals(retrieved.getParamTypes()[0], CatalogUtil.newSimpleDataType(Type.ANY)); } }
http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-client/src/main/java/org/apache/tajo/cli/DescFunctionCommand.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/DescFunctionCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/DescFunctionCommand.java index c20840b..c72f48d 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/DescFunctionCommand.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/DescFunctionCommand.java @@ -18,12 +18,13 @@ package org.apache.tajo.cli; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.function.FunctionUtil; import java.util.*; +import static org.apache.tajo.common.TajoDataTypes.DataType; + public class DescFunctionCommand extends TajoShellCommand { public DescFunctionCommand(TajoCli.TajoCliContext context) { super(context); @@ -53,11 +54,11 @@ public class DescFunctionCommand extends TajoShellCommand { Collections.sort(functions, new Comparator<CatalogProtos.FunctionDescProto>() { @Override public int compare(CatalogProtos.FunctionDescProto f1, CatalogProtos.FunctionDescProto f2) { - int nameCompared = f1.getSignature().compareTo(f2.getSignature()); + int nameCompared = f1.getSignature().getName().compareTo(f2.getSignature().getName()); if (nameCompared != 0) { return nameCompared; } else { - return f1.getReturnType().getType().compareTo(f2.getReturnType().getType()); + return f1.getSignature().getReturnType().getType().compareTo(f2.getSignature().getReturnType().getType()); } } }); @@ -67,11 +68,13 @@ public class DescFunctionCommand extends TajoShellCommand { int[] columnWidths = printHeader(headers, columnWidthRates); for(CatalogProtos.FunctionDescProto eachFunction: functions) { - String name = eachFunction.getSignature(); - String resultDataType = eachFunction.getReturnType().getType().toString(); - String arguments = FunctionDesc.dataTypesToStr(eachFunction.getParameterTypesList()); - String functionType = eachFunction.getType().toString(); - String description = eachFunction.getDescription(); + String name = eachFunction.getSignature().getName(); + String resultDataType = eachFunction.getSignature().getReturnType().getType().toString(); + String arguments = FunctionUtil.buildParamTypeString( + eachFunction.getSignature().getParameterTypesList().toArray( + new DataType[eachFunction.getSignature().getParameterTypesCount()])); + String functionType = eachFunction.getSignature().getType().toString(); + String description = eachFunction.getSupplement().getShortDescription(); int index = 0; printLeft(" " + name, columnWidths[index++]); @@ -96,22 +99,23 @@ public class DescFunctionCommand extends TajoShellCommand { new HashMap<String, CatalogProtos.FunctionDescProto>(); for (CatalogProtos.FunctionDescProto eachFunction: functions) { - if (!functionMap.containsKey(eachFunction.getDescription())) { - functionMap.put(eachFunction.getDescription(), eachFunction); + if (!functionMap.containsKey(eachFunction.getSupplement().getShortDescription())) { + functionMap.put(eachFunction.getSupplement().getShortDescription(), eachFunction); } } for (CatalogProtos.FunctionDescProto eachFunction: functionMap.values()) { - String signature = eachFunction.getReturnType().getType() + " " + - CatalogUtil.getCanonicalSignature(eachFunction.getSignature(), eachFunction.getParameterTypesList()); - String fullDescription = eachFunction.getDescription(); - if(eachFunction.getDetail() != null && !eachFunction.getDetail().isEmpty()) { - fullDescription += "\n" + eachFunction.getDetail(); + String signature = eachFunction.getSignature().getReturnType().getType() + " " + + FunctionUtil.buildSimpleFunctionSignature(eachFunction.getSignature().getName(), + eachFunction.getSignature().getParameterTypesList()); + String fullDescription = eachFunction.getSupplement().getShortDescription(); + if(eachFunction.getSupplement().getDetail() != null && !eachFunction.getSupplement().getDetail().isEmpty()) { + fullDescription += "\n" + eachFunction.getSupplement().getDetail(); } context.getOutput().println("Function: " + signature); context.getOutput().println("Description: " + fullDescription); - context.getOutput().println("Example:\n" + eachFunction.getExample()); + context.getOutput().println("Example:\n" + eachFunction.getSupplement().getExample()); println(); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-common/src/main/java/org/apache/tajo/json/ClassNameSerializer.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/json/ClassNameSerializer.java b/tajo-common/src/main/java/org/apache/tajo/json/ClassNameSerializer.java index 9e9a4ee..c7ec7ae 100644 --- a/tajo-common/src/main/java/org/apache/tajo/json/ClassNameSerializer.java +++ b/tajo-common/src/main/java/org/apache/tajo/json/ClassNameSerializer.java @@ -22,6 +22,7 @@ package org.apache.tajo.json; import com.google.gson.*; +import org.apache.tajo.util.ClassUtil; import java.lang.reflect.Type; @@ -37,7 +38,7 @@ public class ClassNameSerializer implements GsonSerDerAdapter<Class> { public Class deserialize(JsonElement json, Type type, JsonDeserializationContext ctx) throws JsonParseException { try { - return Class.forName(json.getAsString()); + return ClassUtil.forName(json.getAsString()); } catch (ClassNotFoundException e) { e.printStackTrace(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-common/src/main/java/org/apache/tajo/util/ClassUtil.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/util/ClassUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/ClassUtil.java new file mode 100644 index 0000000..6a42b64 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/util/ClassUtil.java @@ -0,0 +1,201 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.util; + +import org.apache.commons.collections.Predicate; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.annotation.Nullable; + +import java.io.File; +import java.util.Enumeration; +import java.util.HashSet; +import java.util.Set; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +public abstract class ClassUtil { + private static final Log LOG = LogFactory.getLog(ClassUtil.class); + + public static Set<Class> findClasses(@Nullable Class targetClass, String packageFilter) { + return findClasses(targetClass, packageFilter, null); + } + + public static Set<Class> findClasses(@Nullable Class targetClass, String packageFilter, Predicate predicate) { + Set<Class> classSet = new HashSet<Class>(); + + String classpath = System.getProperty("java.class.path"); + String[] paths = classpath.split(System.getProperty("path.separator")); + + for (String path : paths) { + File file = new File(path); + if (file.exists()) { + findClasses(classSet, file, file, true, targetClass, packageFilter, predicate); + } + } + + return classSet; + } + + private static void findClasses(Set<Class> matchedClassSet, File root, File file, boolean includeJars, + @Nullable Class type, String packageFilter, @Nullable Predicate predicate) { + if (file.isDirectory()) { + for (File child : file.listFiles()) { + findClasses(matchedClassSet, root, child, includeJars, type, packageFilter, predicate); + } + } else { + if (file.getName().toLowerCase().endsWith(".jar") && includeJars) { + JarFile jar = null; + try { + jar = new JarFile(file); + } catch (Exception ex) { + LOG.error(ex.getMessage(), ex); + return; + } + Enumeration<JarEntry> entries = jar.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + String name = entry.getName(); + int extIndex = name.lastIndexOf(".class"); + if (extIndex > 0) { + String qualifiedClassName = name.substring(0, extIndex).replace("/", "."); + if (qualifiedClassName.indexOf(packageFilter) >= 0 && !isTestClass(qualifiedClassName)) { + try { + Class clazz = Class.forName(qualifiedClassName); + + if (isMatched(clazz, type, predicate)) { + matchedClassSet.add(clazz); + } + } catch (ClassNotFoundException e) { + LOG.error(e.getMessage(), e); + } + } + } + } + } else if (file.getName().toLowerCase().endsWith(".class")) { + String qualifiedClassName = createClassName(root, file); + if (qualifiedClassName.indexOf(packageFilter) >= 0 && !isTestClass(qualifiedClassName)) { + try { + Class clazz = Class.forName(qualifiedClassName); + if (isMatched(clazz, type, predicate)) { + matchedClassSet.add(clazz); + } + } catch (ClassNotFoundException e) { + LOG.error(e.getMessage(), e); + } + } + } + } + } + + private static boolean isMatched(Class clazz, Class targetClass, Predicate predicate) { + return + !clazz.isInterface() && + (targetClass == null || isClassMatched(targetClass, clazz)) && + (predicate == null || predicate.evaluate(clazz)); + } + + private static boolean isTestClass(String qualifiedClassName) { + String className = getSimpleClassName(qualifiedClassName); + if(className == null) { + return false; + } + + return className.startsWith("Test"); + } + + private static boolean isClassMatched(Class targetClass, Class loadedClass) { + if (targetClass.equals(loadedClass)) { + return true; + } + + Class[] classInterfaces = loadedClass.getInterfaces(); + if (classInterfaces != null) { + for (Class eachInterfaceClass : classInterfaces) { + if (eachInterfaceClass.equals(targetClass)) { + return true; + } + + if (isClassMatched(targetClass, eachInterfaceClass)) { + return true; + } + } + } + + Class superClass = loadedClass.getSuperclass(); + if (superClass != null) { + if (isClassMatched(targetClass, superClass)) { + return true; + } + } + return false; + } + + private static String getSimpleClassName(String qualifiedClassName) { + String[] tokens = qualifiedClassName.split("\\."); + if (tokens.length == 0) { + return qualifiedClassName; + } + return tokens[tokens.length - 1]; + } + + private static String createClassName(File root, File file) { + StringBuffer sb = new StringBuffer(); + String fileName = file.getName(); + sb.append(fileName.substring(0, fileName.lastIndexOf(".class"))); + file = file.getParentFile(); + while (file != null && !file.equals(root)) { + sb.insert(0, '.').insert(0, file.getName()); + file = file.getParentFile(); + } + return sb.toString(); + } + + public static Class<?> forName(String name) throws ClassNotFoundException { + if (name.equals("byte")) { + return byte.class; + } + if (name.equals("short")) { + return short.class; + } + if (name.equals("int")) { + return int.class; + } + if (name.equals("long")) { + return long.class; + } + if (name.equals("char")) { + return char.class; + } + if (name.equals("float")) { + return float.class; + } + if (name.equals("double")) { + return double.class; + } + if (name.equals("boolean")) { + return boolean.class; + } + if (name.equals("void")) { + return void.class; + } + + return Class.forName(name); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java index 832c1e5..0ceb2b2 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java @@ -199,14 +199,14 @@ public class TUtil { } } - public static String collectionToString(Collection objects) { + public static String collectionToString(Collection objects, String delimiter) { boolean first = true; StringBuilder sb = new StringBuilder(); for(Object object : objects) { if (first) { first = false; } else { - sb.append(", "); + sb.append(delimiter); } sb.append(object.toString()); http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CaseWhenEmitter.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CaseWhenEmitter.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CaseWhenEmitter.java index 16bd396..9e45e7d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CaseWhenEmitter.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CaseWhenEmitter.java @@ -29,20 +29,15 @@ import org.apache.tajo.org.objectweb.asm.Opcodes; import java.util.List; import java.util.Stack; -class CaseWhenEmitter implements EvalCodeEmitter<CaseWhenEval> { - public static final CaseWhenEmitter instance; - - static { - instance = new CaseWhenEmitter(); - } - - public static CaseWhenEmitter getInstance() { - return instance; - } +/** + * It generates case when byte code. + * + * @see org.apache.tajo.function.StaticMethodInvocationDesc + */ +class CaseWhenEmitter { - @Override - public void emit(EvalCodeGenerator codeGen, EvalCodeGenContext context, CaseWhenEval caseWhen, - Stack<EvalNode> stack) { + public static void emit(EvalCodeGenerator codeGen, EvalCodeGenContext context, CaseWhenEval caseWhen, + Stack<EvalNode> stack) { // TYPE 1: CASE <expr> WHEN x THEN c1 WHEN y THEN c2 WHEN c3 ELSE ... END; EvalNode commonTerm = extractCommonTerm(caseWhen.getIfThenEvals()); @@ -144,7 +139,7 @@ class CaseWhenEmitter implements EvalCodeEmitter<CaseWhenEval> { } } - private EvalNode extractCommonTerm(List<CaseWhenEval.IfThenEval> ifThenEvals) { + private static EvalNode extractCommonTerm(List<CaseWhenEval.IfThenEval> ifThenEvals) { EvalNode commonTerm = null; for (int i = 0; i < ifThenEvals.size(); i++) { @@ -180,7 +175,7 @@ class CaseWhenEmitter implements EvalCodeEmitter<CaseWhenEval> { * @param predicate Predicate to be checked * @return True if the predicate is a simple form. */ - private boolean checkIfSimplePredicate(EvalNode predicate) { + private static boolean checkIfSimplePredicate(EvalNode predicate) { if (predicate.getType() == EvalType.EQUAL) { BinaryEval binaryEval = (BinaryEval) predicate; EvalNode lhs = binaryEval.getLeftExpr(); @@ -195,7 +190,7 @@ class CaseWhenEmitter implements EvalCodeEmitter<CaseWhenEval> { } } - private int getSwitchIndex(EvalNode predicate) { + private static int getSwitchIndex(EvalNode predicate) { Preconditions.checkArgument(checkIfSimplePredicate(predicate), "This expression cannot be used for switch table: " + predicate); http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java index c57f923..147b149 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java @@ -19,7 +19,6 @@ package org.apache.tajo.engine.codegen; import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; @@ -27,16 +26,18 @@ import org.apache.tajo.datum.IntervalDatum; import org.apache.tajo.datum.ProtobufDatum; import org.apache.tajo.engine.eval.*; import org.apache.tajo.engine.json.CoreGsonHelper; -import org.apache.tajo.org.objectweb.asm.*; +import org.apache.tajo.org.objectweb.asm.ClassWriter; +import org.apache.tajo.org.objectweb.asm.Label; +import org.apache.tajo.org.objectweb.asm.Opcodes; +import org.apache.tajo.org.objectweb.asm.Type; import org.apache.tajo.storage.Tuple; -import org.apache.tajo.storage.VTuple; import java.io.PrintStream; import java.lang.reflect.Constructor; import java.util.Stack; import static org.apache.tajo.common.TajoDataTypes.DataType; -import static org.apache.tajo.engine.codegen.TajoGeneratorAdapter.*; +import static org.apache.tajo.engine.codegen.TajoGeneratorAdapter.getDescription; import static org.apache.tajo.engine.eval.FunctionEval.ParamType; public class EvalCodeGenerator extends SimpleEvalNodeVisitor<EvalCodeGenContext> { @@ -339,7 +340,6 @@ public class EvalCodeGenerator extends SimpleEvalNodeVisitor<EvalCodeGenContext> context.pop(srcType); context.pushDummyValue(targetType); context.pushNullFlag(false); - printOut(context, "endIfNull"); emitLabel(context, afterEnd); return cast; @@ -719,40 +719,17 @@ public class EvalCodeGenerator extends SimpleEvalNodeVisitor<EvalCodeGenContext> @Override public EvalNode visitFuncCall(EvalCodeGenContext context, FunctionEval func, Stack<EvalNode> stack) { - int paramNum = func.getArgs().length; - context.push(paramNum); - context.newArray(Datum.class); // new Datum[paramNum] - final int DATUM_ARRAY = context.astore(); - - stack.push(func); - EvalNode [] params = func.getArgs(); - for (int paramIdx = 0; paramIdx < func.getArgs().length; paramIdx++) { - context.aload(DATUM_ARRAY); // array ref - context.methodvisitor.visitLdcInsn(paramIdx); // array idx - visit(context, params[paramIdx], stack); - context.convertToDatum(params[paramIdx].getValueType(), true); // value - context.methodvisitor.visitInsn(Opcodes.AASTORE); - } - stack.pop(); - - context.methodvisitor.visitTypeInsn(Opcodes.NEW, TajoGeneratorAdapter.getInternalName(VTuple.class)); - context.methodvisitor.visitInsn(Opcodes.DUP); - context.aload(DATUM_ARRAY); - context.newInstance(VTuple.class, new Class[]{Datum[].class}); // new VTuple(datum []) - context.methodvisitor.visitTypeInsn(Opcodes.CHECKCAST, TajoGeneratorAdapter.getInternalName(Tuple.class)); // cast to Tuple - final int TUPLE = context.astore(); - - FunctionDesc desc = func.getFuncDesc(); - String fieldName = context.symbols.get(func); - String funcDescName = "L" + TajoGeneratorAdapter.getInternalName(desc.getFuncClass()) + ";"; + if (func.getFuncDesc().getInvocation().hasScalar()) { + ScalarFunctionBindingEmitter.emit(this, context, func, stack); + return func; + } - context.aload(0); - context.methodvisitor.visitFieldInsn(Opcodes.GETFIELD, context.owner, fieldName, funcDescName); - context.aload(TUPLE); - context.invokeVirtual(desc.getFuncClass(), "eval", Datum.class, new Class[] {Tuple.class}); + if (func.getFuncDesc().getInvocation().hasLegacy()) { + LegacyFunctionBindingEmitter.emit(this, context, func, stack); + return func; + } - context.convertToPrimitive(func.getValueType()); return func; } @@ -824,7 +801,7 @@ public class EvalCodeGenerator extends SimpleEvalNodeVisitor<EvalCodeGenContext> @Override protected EvalNode visitCaseWhen(EvalCodeGenContext context, CaseWhenEval caseWhen, Stack<EvalNode> stack) { - CaseWhenEmitter.getInstance().emit(this, context, caseWhen, stack); + CaseWhenEmitter.emit(this, context, caseWhen, stack); return caseWhen; } http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/codegen/LegacyFunctionBindingEmitter.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/LegacyFunctionBindingEmitter.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/LegacyFunctionBindingEmitter.java new file mode 100644 index 0000000..944a01c --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/LegacyFunctionBindingEmitter.java @@ -0,0 +1,75 @@ +/*** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.codegen; + +import org.apache.tajo.catalog.FunctionDesc; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.engine.eval.EvalNode; +import org.apache.tajo.engine.eval.FunctionEval; +import org.apache.tajo.org.objectweb.asm.Opcodes; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.VTuple; + +import java.util.Stack; + +/** + * It generates the legacy function binding code for GeneralFunction and AggFunction. + * + * @see org.apache.tajo.function.StaticMethodInvocationDesc + */ +public class LegacyFunctionBindingEmitter { + + public static void emit(EvalCodeGenerator generator, EvalCodeGenContext context, FunctionEval func, + Stack<EvalNode> stack) { + int paramNum = func.getArgs().length; + context.push(paramNum); + context.newArray(Datum.class); // new Datum[paramNum] + final int DATUM_ARRAY = context.astore(); + + stack.push(func); + EvalNode [] params = func.getArgs(); + for (int paramIdx = 0; paramIdx < func.getArgs().length; paramIdx++) { + context.aload(DATUM_ARRAY); // array ref + context.methodvisitor.visitLdcInsn(paramIdx); // array idx + generator.visit(context, params[paramIdx], stack); + context.convertToDatum(params[paramIdx].getValueType(), true); // value + context.methodvisitor.visitInsn(Opcodes.AASTORE); + } + stack.pop(); + + context.methodvisitor.visitTypeInsn(Opcodes.NEW, TajoGeneratorAdapter.getInternalName(VTuple.class)); + context.methodvisitor.visitInsn(Opcodes.DUP); + context.aload(DATUM_ARRAY); + context.newInstance(VTuple.class, new Class[]{Datum[].class}); // new VTuple(datum []) + context.methodvisitor.visitTypeInsn(Opcodes.CHECKCAST, TajoGeneratorAdapter.getInternalName(Tuple.class)); // cast to Tuple + final int TUPLE = context.astore(); + + FunctionDesc desc = func.getFuncDesc(); + + String fieldName = context.symbols.get(func); + String funcDescName = "L" + TajoGeneratorAdapter.getInternalName(desc.getFuncClass()) + ";"; + + context.aload(0); + context.methodvisitor.visitFieldInsn(Opcodes.GETFIELD, context.owner, fieldName, funcDescName); + context.aload(TUPLE); + context.invokeVirtual(desc.getFuncClass(), "eval", Datum.class, new Class[] {Tuple.class}); + + context.convertToPrimitive(func.getValueType()); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ScalarFunctionBindingEmitter.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ScalarFunctionBindingEmitter.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ScalarFunctionBindingEmitter.java new file mode 100644 index 0000000..350aee1 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ScalarFunctionBindingEmitter.java @@ -0,0 +1,164 @@ +/*** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.codegen; + +import com.google.common.base.Preconditions; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.engine.eval.EvalNode; +import org.apache.tajo.engine.eval.FunctionEval; +import org.apache.tajo.function.FunctionUtil; +import org.apache.tajo.function.StaticMethodInvocationDesc; +import org.apache.tajo.org.objectweb.asm.Label; +import org.apache.tajo.org.objectweb.asm.Opcodes; + +import java.util.Stack; + +/** + * It generates the scalar function binding code for StaticMethodInvocationDesc. + * + * @see org.apache.tajo.function.StaticMethodInvocationDesc + */ +public class ScalarFunctionBindingEmitter { + + public static void emit(EvalCodeGenerator generator, EvalCodeGenContext context, FunctionEval func, + Stack<EvalNode> stack) { + + EvalNode [] params = func.getArgs(); + + StaticMethodInvocationDesc method = func.getFuncDesc().getInvocation().getScalar(); + + // check if there are at least one nullable field. + int notNullParamNum = 0; + for (Class paramClass : method.getParamClasses()) { + notNullParamNum += !FunctionUtil.isNullableParam(paramClass) ? 1 : 0; + } + + int nullParamFlag = -1; + if (notNullParamNum > 0) { + // initialize the base null flag + context.methodvisitor.visitInsn(Opcodes.ICONST_1); + nullParamFlag = context.istore(); + } + + stack.push(func); + for (int paramIdx = 0; paramIdx < func.getArgs().length; paramIdx++) { + Class clazz = method.getParamClasses()[paramIdx]; + + generator.visit(context, params[paramIdx], stack); + + if (FunctionUtil.isNullableParam(clazz)) { + emitBoxedParameter(context, func.getArgs()[paramIdx].getValueType()); + } else { + updateNullFlag(context, clazz, nullParamFlag); + } + } + stack.pop(); + + if (notNullParamNum > 0) { + Label ifNull = new Label(); + Label afterAll = new Label(); + + Preconditions.checkArgument(nullParamFlag >= 0); + context.iload(nullParamFlag); + context.methodvisitor.visitJumpInsn(Opcodes.IFEQ, ifNull); + + // -- If all parameters are NOT NULL + context.invokeStatic( + method.getBaseClassName(), + method.getMethodName(), + method.getReturnClass(), + method.getParamClasses()); + emitFunctionReturnValue(context, func.getValueType(), method); + context.gotoLabel(afterAll); + + // -- If at least parameter is NULL + context.methodvisitor.visitLabel(ifNull); + + for (int paramIdx = 0; paramIdx < func.getArgs().length; paramIdx++) { + Class clazz = method.getParamClasses()[paramIdx]; + if (FunctionUtil.isNullableParam(clazz)) { + context.pop(); + } else { + context.pop(func.getArgs()[paramIdx].getValueType()); + } + } + context.pushDummyValue(func.getValueType()); + context.pushNullFlag(false); + + // -- After All + context.methodvisitor.visitLabel(afterAll); + + } else { + context.invokeStatic( + method.getBaseClassName(), + method.getMethodName(), + method.getReturnClass(), + method.getParamClasses()); + emitFunctionReturnValue(context, func.getValueType(), method); + } + } + + private static void emitFunctionReturnValue(EvalCodeGenContext context, TajoDataTypes.DataType returnType, + StaticMethodInvocationDesc method) { + if (FunctionUtil.isNullableParam(method.getReturnClass())) { + Label ifNull = new Label(); + Label afterAll = new Label(); + + context.dup(); + context.methodvisitor.visitJumpInsn(Opcodes.IFNULL, ifNull); + + context.emitUnboxing(context, returnType); + context.pushNullFlag(true); // push null flag + context.gotoLabel(afterAll); + + context.markLabel(ifNull); + context.pop(); // remove null reference + context.pushDummyValue(returnType); // push dummy value for stack balance + context.pushNullFlag(false); // push null flag + + context.markLabel(afterAll); + } else { + context.pushNullFlag(true); + } + } + + private static void updateNullFlag(EvalCodeGenContext context, Class clazz, int nullFlagId) { + Preconditions.checkArgument(!FunctionUtil.isNullableParam(clazz)); + context.iload(nullFlagId); + context.methodvisitor.visitInsn(Opcodes.IAND); + context.istore(nullFlagId); + } + + private static void emitBoxedParameter(EvalCodeGenContext context, TajoDataTypes.DataType dataType) { + Label ifNull = new Label(); + Label afterAll = new Label(); + + context.emitNullityCheck(ifNull); + + context.emitBoxing(context, dataType); + context.gotoLabel(afterAll); + + context.markLabel(ifNull); + context.pop(dataType); // pop dummy value + context.methodvisitor.visitInsn(Opcodes.ACONST_NULL); + + context.methodvisitor.visitLabel(afterAll); + } + +} http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java index 6fac1a8..57dc904 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java @@ -354,6 +354,10 @@ class TajoGeneratorAdapter { return new Label(); } + public void markLabel(Label label) { + methodvisitor.visitLabel(label); + } + public void gotoLabel(Label label) { methodvisitor.visitJumpInsn(Opcodes.GOTO, label); } @@ -864,6 +868,10 @@ class TajoGeneratorAdapter { public int istore() { int varId = getCurVarIdAndIncrease(); + return istore(varId); + } + + public int istore(int varId) { methodvisitor.visitVarInsn(Opcodes.ISTORE, varId); return varId; } @@ -919,6 +927,58 @@ class TajoGeneratorAdapter { return varId; } + public void emitBoxing(EvalCodeGenContext context, TajoDataTypes.DataType dataType) { + switch (dataType.getType()) { + case CHAR: + case TEXT: + + case INT2: + context.invokeStatic(Short.class, "valueOf", Short.class, new Class[]{short.class}); + break; + case INT4: + context.invokeStatic(Integer.class, "valueOf", Integer.class, new Class[]{int.class}); + break; + case INT8: + context.invokeStatic(Long.class, "valueOf", Long.class, new Class[]{long.class}); + break; + case FLOAT4: + context.invokeStatic(Float.class, "valueOf", Float.class, new Class[]{float.class}); + break; + case FLOAT8: + context.invokeStatic(Double.class, "valueOf", Double.class, new Class[]{double.class}); + break; + + default: + throw new RuntimeException(dataType.getType().name() + " is not supported yet"); + } + } + + public void emitUnboxing(EvalCodeGenContext context, TajoDataTypes.DataType dataType) { + switch (dataType.getType()) { + case CHAR: + case TEXT: + + case INT2: + context.invokeVirtual(Short.class, "shortValue", short.class, new Class[]{}); + break; + case INT4: + context.invokeVirtual(Integer.class, "intValue", int.class, new Class[]{}); + break; + case INT8: + context.invokeVirtual(Long.class, "longValue", long.class, new Class[]{}); + break; + case FLOAT4: + context.invokeVirtual(Float.class, "floatValue", float.class, new Class[]{}); + break; + case FLOAT8: + context.invokeVirtual(Double.class, "doubleValue", double.class, new Class[]{}); + break; + + default: + throw new RuntimeException(dataType.getType().name() + " is not supported yet"); + } + } + public static interface SwitchCaseGenerator extends TableSwitchGenerator { int size(); int min(); http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/codegen/VariablesPreBuilder.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/VariablesPreBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/VariablesPreBuilder.java index 9f50bb5..c81f242 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/VariablesPreBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/VariablesPreBuilder.java @@ -72,7 +72,7 @@ class VariablesPreBuilder extends SimpleEvalNodeVisitor<EvalCodeGenContext> { super.visitFuncCall(context, function, stack); if (!context.symbols.containsKey(function)) { - String fieldName = function.getFuncDesc().getSignature() + "_" + context.seqId++; + String fieldName = function.getFuncDesc().getFunctionName() + "_" + context.seqId++; context.symbols.put(function, fieldName); context.classWriter.visitField(Opcodes.ACC_PRIVATE, fieldName, "L" + TajoGeneratorAdapter.getInternalName(function.getFuncDesc().getFuncClass()) + ";", null, null); http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicUtil.java index 518b72f..0b4ba19 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicUtil.java @@ -172,7 +172,7 @@ public class AlgebraicUtil { public EvalNode visitFuncCall(Object context, FunctionEval evalNode, Stack<EvalNode> stack) { boolean constantOfAllDescendents = true; - if ("sleep".equals(evalNode.funcDesc.getSignature())) { + if ("sleep".equals(evalNode.funcDesc.getFunctionName())) { constantOfAllDescendents = false; } else { if (evalNode.getArgs() != null) { http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java index d7473e9..85a0d7c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java @@ -26,7 +26,7 @@ import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.utils.DataTypeUtil; +import org.apache.tajo.DataTypeUtil; import org.apache.tajo.exception.InvalidOperationException; import org.apache.tajo.storage.Tuple; http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java index 5d358f7..b449040 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java @@ -107,7 +107,7 @@ public abstract class FunctionEval extends EvalNode implements Cloneable { @Override public String getName() { - return funcDesc.getSignature(); + return funcDesc.getFunctionName(); } @Override @@ -118,7 +118,7 @@ public abstract class FunctionEval extends EvalNode implements Cloneable { if(i+1 < argEvals.length) sb.append(","); } - return funcDesc.getSignature() + "(" + (isDistinct() ? " distinct " : "") + sb+")"; + return funcDesc.getFunctionName() + "(" + (isDistinct() ? " distinct " : "") + sb+")"; } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/eval/WindowFunctionEval.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/WindowFunctionEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/WindowFunctionEval.java index fb4eede..4057e70 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/WindowFunctionEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/WindowFunctionEval.java @@ -107,7 +107,7 @@ public class WindowFunctionEval extends AggregationFunctionCallEval implements C sb.append(","); } } - sb.append(funcDesc.getSignature()).append("(").append(isDistinct() ? " distinct" : "").append(sb) + sb.append(funcDesc.getFunctionName()).append("(").append(isDistinct() ? " distinct" : "").append(sb) .append(")"); if (hasSortSpecs()) { sb.append("ORDER BY ").append(TUtil.arrayToString(sortSpecs)); http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/function/AggFunction.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/AggFunction.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/AggFunction.java index a5a2583..10f4c3f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/AggFunction.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/AggFunction.java @@ -19,13 +19,14 @@ package org.apache.tajo.engine.function; import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.function.Function; +import org.apache.tajo.function.Function; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.Datum; import org.apache.tajo.storage.Tuple; +@Deprecated public abstract class AggFunction<T extends Datum> extends Function<T> { public AggFunction(Column[] definedArgs) { http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java new file mode 100644 index 0000000..334a92b --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java @@ -0,0 +1,239 @@ +/*** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.commons.collections.Predicate; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.annotation.Nullable; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.FunctionDesc; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.engine.function.annotation.Description; +import org.apache.tajo.engine.function.annotation.ParamOptionTypes; +import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.function.*; +import org.apache.tajo.util.ClassUtil; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.*; + +import static org.apache.tajo.catalog.proto.CatalogProtos.FunctionType.GENERAL; + +public class FunctionLoader { + + private static Log LOG = LogFactory.getLog(FunctionLoader.class); + + public static Collection<FunctionDesc> load() { + Map<FunctionSignature, FunctionDesc> map = Maps.newHashMap(); + + List<FunctionDesc> dd = Lists.newArrayList(); + for (FunctionDesc f : findLegacyFunctions()) { + map.put(f.getSignature(), f); + + if (f.getSignature().getName().equals("pow") || f.getSignature().getName().equals("pi")) { + dd.add(f); + } + } + + for (FunctionDesc f : findScalarFunctions()) { + if (map.containsKey(f.getSignature())) { + FunctionDesc existing = map.get(f.getSignature()); + existing.getInvocation().setScalar(f.getInvocation().getScalar()); + } else { + map.put(f.getSignature(), f); + } + } + + return map.values(); + } + + public static Set<FunctionDesc> findScalarFunctions() { + Set<FunctionDesc> functions = Sets.newHashSet(); + + Set<Method> scalarFunctions = findPublicStaticMethods("org.apache.tajo.engine.function", new Predicate() { + @Override + public boolean evaluate(Object object) { + return ((Method) object).getAnnotation(ScalarFunction.class) != null; + } + }); + + for (Method method : scalarFunctions) { + ScalarFunction annotation = method.getAnnotation(ScalarFunction.class); + functions.addAll(buildFunctionDescs(annotation, method)); + } + + return functions; + } + + private static Set<Method> findPublicStaticMethods(String packageName, Predicate predicate) { + Set<Class> found = findFunctionCollections(packageName); + Set<Method> filtered = Sets.newHashSet(); + + for (Class clazz : found) { + for (Method method : clazz.getMethods()) { + if (isPublicStaticMethod(method) && (predicate == null || predicate.evaluate(method))) { + filtered.add(method); + } + } + } + + return filtered; + } + + private static boolean isPublicStaticMethod(Method method) { + return Modifier.isPublic(method.getModifiers()) && Modifier.isStatic(method.getModifiers()); + } + + private static Set<Class> findFunctionCollections(String packageName) { + return ClassUtil.findClasses(null, packageName, new Predicate() { + @Override + public boolean evaluate(Object object) { + return ((Class)object).getAnnotation(FunctionCollection.class) != null; + } + }); + } + + private static Collection<FunctionDesc> buildFunctionDescs(ScalarFunction annotation, Method method) { + List<FunctionDesc> functionDescs = Lists.newArrayList(); + + FunctionInvocation invocation = new FunctionInvocation(); + invocation.setScalar(extractStaticMethodInvocation(method)); + FunctionSupplement supplement = extractSupplement(annotation); + + // primary name + functionDescs.add(new FunctionDesc(extractSignature(annotation, null), invocation, supplement)); + + // for multiple aliases + for (String alias : annotation.synonyms()) { + functionDescs.add(new FunctionDesc(extractSignature(annotation, alias), invocation, supplement)); + } + + return functionDescs; + } + + private static FunctionSignature extractSignature(ScalarFunction annotation, @Nullable String alias) { + return new FunctionSignature( + GENERAL, + alias != null ? alias : annotation.name(), + CatalogUtil.newSimpleDataType(annotation.returnType()), + CatalogUtil.newSimpleDataTypeArray(annotation.paramTypes()) + ); + } + + private static FunctionSupplement extractSupplement(ScalarFunction function) { + return new FunctionSupplement( + function.shortDescription(), + function.detail(), + function.example()); + } + + private static StaticMethodInvocationDesc extractStaticMethodInvocation(Method method) { + Preconditions.checkArgument(Modifier.isPublic(method.getModifiers())); + Preconditions.checkArgument(Modifier.isStatic(method.getModifiers())); + + String methodName = method.getName(); + Class returnClass = method.getReturnType(); + Class [] paramClasses = method.getParameterTypes(); + return new StaticMethodInvocationDesc(method.getDeclaringClass(), methodName, returnClass, paramClasses); + } + + /** + * This method finds and build FunctionDesc for the legacy function and UD(A)F system. + * + * @return A list of FunctionDescs + */ + public static List<FunctionDesc> findLegacyFunctions() { + List<FunctionDesc> sqlFuncs = new ArrayList<FunctionDesc>(); + + Set<Class> functionClasses = ClassUtil.findClasses(Function.class, "org.apache.tajo.engine.function"); + + for (Class eachClass : functionClasses) { + if(eachClass.isInterface() || Modifier.isAbstract(eachClass.getModifiers())) { + continue; + } + Function function = null; + try { + function = (Function)eachClass.newInstance(); + } catch (Exception e) { + LOG.warn(eachClass + " cannot instantiate Function class because of " + e.getMessage()); + continue; + } + String functionName = function.getClass().getAnnotation(Description.class).functionName(); + String[] synonyms = function.getClass().getAnnotation(Description.class).synonyms(); + String description = function.getClass().getAnnotation(Description.class).description(); + String detail = function.getClass().getAnnotation(Description.class).detail(); + String example = function.getClass().getAnnotation(Description.class).example(); + TajoDataTypes.Type returnType = function.getClass().getAnnotation(Description.class).returnType(); + ParamTypes[] paramArray = function.getClass().getAnnotation(Description.class).paramTypes(); + + String[] allFunctionNames = null; + if(synonyms != null && synonyms.length > 0) { + allFunctionNames = new String[1 + synonyms.length]; + allFunctionNames[0] = functionName; + System.arraycopy(synonyms, 0, allFunctionNames, 1, synonyms.length); + } else { + allFunctionNames = new String[]{functionName}; + } + + for(String eachFunctionName: allFunctionNames) { + for (ParamTypes params : paramArray) { + ParamOptionTypes[] paramOptionArray; + if(params.paramOptionTypes() == null || + params.paramOptionTypes().getClass().getAnnotation(ParamTypes.class) == null) { + paramOptionArray = new ParamOptionTypes[0]; + } else { + paramOptionArray = params.paramOptionTypes().getClass().getAnnotation(ParamTypes.class).paramOptionTypes(); + } + + TajoDataTypes.Type[] paramTypes = params.paramTypes(); + if (paramOptionArray.length > 0) + paramTypes = params.paramTypes().clone(); + + for (int i=0; i < paramOptionArray.length + 1; i++) { + FunctionDesc functionDesc = new FunctionDesc(eachFunctionName, + function.getClass(), function.getFunctionType(), + CatalogUtil.newSimpleDataType(returnType), + paramTypes.length == 0 ? CatalogUtil.newSimpleDataTypeArray() : CatalogUtil.newSimpleDataTypeArray(paramTypes)); + + functionDesc.setDescription(description); + functionDesc.setExample(example); + functionDesc.setDetail(detail); + sqlFuncs.add(functionDesc); + + if (i != paramOptionArray.length) { + paramTypes = new TajoDataTypes.Type[paramTypes.length + + paramOptionArray[i].paramOptionTypes().length]; + System.arraycopy(params.paramTypes(), 0, paramTypes, 0, paramTypes.length); + System.arraycopy(paramOptionArray[i].paramOptionTypes(), 0, paramTypes, paramTypes.length, + paramOptionArray[i].paramOptionTypes().length); + } + } + } + } + } + + return sqlFuncs; + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/function/GeneralFunction.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/GeneralFunction.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/GeneralFunction.java index f7c31a1..a97baf0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/GeneralFunction.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/GeneralFunction.java @@ -19,7 +19,7 @@ package org.apache.tajo.engine.function; import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.function.Function; +import org.apache.tajo.function.Function; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.datum.Datum; @@ -27,6 +27,7 @@ import org.apache.tajo.engine.eval.FunctionEval; import org.apache.tajo.json.GsonObject; import org.apache.tajo.storage.Tuple; +@Deprecated public abstract class GeneralFunction extends Function implements GsonObject { public GeneralFunction(Column[] definedArgs) { super(definedArgs); http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgInt.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgInt.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgInt.java index 3c59f13..192b1a6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgInt.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgInt.java @@ -43,5 +43,4 @@ public class AvgInt extends AvgLong { avgCtx.sum += params.get(0).asInt4(); avgCtx.count++; } - } http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/function/math/MathFunctions.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/MathFunctions.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/MathFunctions.java new file mode 100644 index 0000000..ee27ba6 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/MathFunctions.java @@ -0,0 +1,46 @@ +/*** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function.math; + +import org.apache.tajo.function.FunctionCollection; +import org.apache.tajo.function.ScalarFunction; + +import static org.apache.tajo.common.TajoDataTypes.Type.FLOAT8; + +@FunctionCollection +public class MathFunctions { + + @ScalarFunction(name = "pi", returnType = FLOAT8) + public static double pi() { + return Math.PI; + } + + @ScalarFunction(name = "pow", returnType = FLOAT8, paramTypes = {FLOAT8, FLOAT8}) + public static Double pow(Double x, Double y) { + if (x == null || y == null) { + return null; + } + return Math.pow(x, y); + } + +// @ScalarFunction(name = "pow", returnType = FLOAT8, paramTypes = {FLOAT8, FLOAT8}) +// public static double pow(double x, double y) { +// return Math.pow(x, y); +// } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Pow.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Pow.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Pow.java index 101e508..269fec8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Pow.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Pow.java @@ -28,6 +28,8 @@ import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; +import static org.apache.tajo.common.TajoDataTypes.Type.FLOAT8; + /** * Function definition * @@ -38,30 +40,16 @@ import org.apache.tajo.storage.Tuple; description = "x raised to the power of y", example = "> SELECT pow(9.0, 3.0)\n" + "729", - returnType = TajoDataTypes.Type.FLOAT8, - paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.FLOAT4, TajoDataTypes.Type.FLOAT4}), - @ParamTypes(paramTypes = {TajoDataTypes.Type.FLOAT4, TajoDataTypes.Type.FLOAT8}), - @ParamTypes(paramTypes = {TajoDataTypes.Type.FLOAT8, TajoDataTypes.Type.FLOAT4}), - @ParamTypes(paramTypes = {TajoDataTypes.Type.FLOAT8, TajoDataTypes.Type.FLOAT8}), - @ParamTypes(paramTypes = {TajoDataTypes.Type.INT4, TajoDataTypes.Type.INT4}), - @ParamTypes(paramTypes = {TajoDataTypes.Type.INT4, TajoDataTypes.Type.INT8}), - @ParamTypes(paramTypes = {TajoDataTypes.Type.INT8, TajoDataTypes.Type.INT4}), - @ParamTypes(paramTypes = {TajoDataTypes.Type.INT8, TajoDataTypes.Type.INT8}), - @ParamTypes(paramTypes = {TajoDataTypes.Type.INT4, TajoDataTypes.Type.FLOAT4}), - @ParamTypes(paramTypes = {TajoDataTypes.Type.INT4, TajoDataTypes.Type.FLOAT8}), - @ParamTypes(paramTypes = {TajoDataTypes.Type.INT8, TajoDataTypes.Type.FLOAT4}), - @ParamTypes(paramTypes = {TajoDataTypes.Type.INT8, TajoDataTypes.Type.FLOAT8}), - @ParamTypes(paramTypes = {TajoDataTypes.Type.FLOAT4, TajoDataTypes.Type.INT4}), - @ParamTypes(paramTypes = {TajoDataTypes.Type.FLOAT4, TajoDataTypes.Type.INT8}), - @ParamTypes(paramTypes = {TajoDataTypes.Type.FLOAT8, TajoDataTypes.Type.INT4}), - @ParamTypes(paramTypes = {TajoDataTypes.Type.FLOAT8, TajoDataTypes.Type.INT8}) + returnType = FLOAT8, + paramTypes = { + @ParamTypes(paramTypes = {FLOAT8, FLOAT8}) } ) public class Pow extends GeneralFunction { public Pow() { super(new Column[] { - new Column("x", TajoDataTypes.Type.FLOAT8), - new Column("y", TajoDataTypes.Type.FLOAT8) + new Column("x", FLOAT8), + new Column("y", FLOAT8) }); } http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java b/tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java index 4dfb314..0340a32 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java @@ -22,7 +22,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.function.Function; +import org.apache.tajo.function.Function; import org.apache.tajo.catalog.json.FunctionAdapter; import org.apache.tajo.catalog.json.TableMetaAdapter; import org.apache.tajo.common.TajoDataTypes.DataType; http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/rules/ConstantFolding.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/rules/ConstantFolding.java b/tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/rules/ConstantFolding.java index b327e56..8bcb0cd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/rules/ConstantFolding.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/rules/ConstantFolding.java @@ -72,7 +72,7 @@ public class ConstantFolding extends SimpleEvalNodeVisitor<LogicalPlanner.PlanCo public EvalNode visitFuncCall(LogicalPlanner.PlanContext context, FunctionEval evalNode, Stack<EvalNode> stack) { boolean constantOfAllDescendents = true; - if ("sleep".equals(evalNode.getFuncDesc().getSignature())) { + if ("sleep".equals(evalNode.getFuncDesc().getFunctionName())) { constantOfAllDescendents = false; } else { if (evalNode.getArgs() != null) { http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoDeserializer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoDeserializer.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoDeserializer.java index 0fa6dbb..b62e236 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoDeserializer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoDeserializer.java @@ -164,9 +164,9 @@ public class EvalTreeProtoDeserializer { } } } catch (ClassNotFoundException cnfe) { - throw new NoSuchFunctionException(funcDesc.getSignature(), funcDesc.getParamTypes()); + throw new NoSuchFunctionException(funcDesc.getFunctionName(), funcDesc.getParamTypes()); } catch (InternalException ie) { - throw new NoSuchFunctionException(funcDesc.getSignature(), funcDesc.getParamTypes()); + throw new NoSuchFunctionException(funcDesc.getFunctionName(), funcDesc.getParamTypes()); } } else { throw new RuntimeException("Unknown EvalType: " + type.name()); http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java index 574e32c..3115104 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java @@ -18,7 +18,6 @@ package org.apache.tajo.engine.planner; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.CatalogService; @@ -35,12 +34,12 @@ import org.apache.tajo.engine.planner.logical.NodeType; import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; import org.apache.tajo.engine.planner.nameresolver.NameResolver; import org.apache.tajo.exception.InternalException; +import org.apache.tajo.exception.InvalidOperationException; import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.datetime.DateTimeUtil; import org.apache.tajo.util.datetime.TimeMeta; -import java.util.Map; import java.util.Set; import java.util.Stack; @@ -106,7 +105,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva return new Pair<EvalNode, EvalNode>(lhs, rhs); } - Type toBeCasted = TUtil.getFromNestedMap(TYPE_CONVERSION_MAP, lhsType, rhsType); + Type toBeCasted = TUtil.getFromNestedMap(CatalogUtil.OPERATION_CASTING_MAP, lhsType, rhsType); if (toBeCasted != null) { // if not null, one of either should be converted to another type. // Overwrite lhs, rhs, or both with cast expression. if (lhsType != toBeCasted) { @@ -121,40 +120,6 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva } /** - * It picks out the widest range type among given <code>types</code>. - * - * Example: - * <ul> - * <li>int, int8 -> int8 </li> - * <li>int4, int8, float4 -> float4 </li> - * <li>float4, float8 -> float8</li> - * <li>float4, text -> exception!</li> - * </ul> - * - * @param types A list of DataTypes - * @return The widest DataType - * @throws PlanningException when types are not compatible, it throws the exception. - */ - public static DataType getWidestType(DataType...types) throws PlanningException { - DataType widest = types[0]; - for (int i = 1; i < types.length; i++) { - - if (widest.getType() == Type.NULL_TYPE) { // if null, skip this type - widest = types[i]; - continue; - } - - if (types[i].getType() != Type.NULL_TYPE) { - Type candidate = TUtil.getFromNestedMap(TYPE_CONVERSION_MAP, widest.getType(), types[i].getType()); - assertEval(candidate != null, "No matched operation for those types: " + TUtil.arrayToString(types)); - widest = CatalogUtil.newSimpleDataType(candidate); - } - } - - return widest; - } - - /** * Insert a type conversion expression to a given expression. * If the type of expression and <code>toType</code> is already the same, it just returns the original expression. * @@ -331,7 +296,13 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva stack.pop(); // implicit type conversion - DataType widestType = getWidestType(predicand.getValueType(), begin.getValueType(), end.getValueType()); + DataType widestType = null; + + try { + widestType = CatalogUtil.getWidestType(predicand.getValueType(), begin.getValueType(), end.getValueType()); + } catch (InvalidOperationException ioe) { + throw new PlanningException(ioe); + } BetweenPredicateEval betweenEval = new BetweenPredicateEval( between.isNot(), @@ -362,10 +333,11 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva // Getting the widest type from all if-then expressions and else expression. DataType widestType = caseWhenEval.getIfThenEvals().get(0).getResult().getValueType(); for (int i = 1; i < caseWhenEval.getIfThenEvals().size(); i++) { - widestType = getWidestType(caseWhenEval.getIfThenEvals().get(i).getResult().getValueType(), widestType); + widestType = CatalogUtil.getWidestType(caseWhenEval.getIfThenEvals().get(i).getResult().getValueType(), + widestType); } if (caseWhen.hasElseResult()) { - widestType = getWidestType(widestType, caseWhenEval.getElse().getValueType()); + widestType = CatalogUtil.getWidestType(widestType, caseWhenEval.getElse().getValueType()); } assertEval(widestType != null, "Invalid Type Conversion for CaseWhen"); @@ -662,7 +634,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva return new AggregationFunctionCallEval(countRows, (AggFunction) countRows.newInstance(), new EvalNode[] {}); } catch (InternalException e) { - throw new NoSuchFunctionException(countRows.getSignature(), new DataType[]{}); + throw new NoSuchFunctionException(countRows.getFunctionName(), new DataType[]{}); } } @@ -960,63 +932,4 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva return results; } - - /** It is the relationship graph of type conversions. It represents each type can be converted to which types. */ - static final Map<Type, Map<Type, Type>> TYPE_CONVERSION_MAP = Maps.newHashMap(); - static { - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT1, Type.INT1, Type.INT1); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT1, Type.INT2, Type.INT2); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT1, Type.INT4, Type.INT4); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT1, Type.INT8, Type.INT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT1, Type.FLOAT4, Type.FLOAT4); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT1, Type.FLOAT8, Type.FLOAT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT1, Type.TEXT, Type.TEXT); - - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT2, Type.INT1, Type.INT2); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT2, Type.INT2, Type.INT2); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT2, Type.INT4, Type.INT4); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT2, Type.INT8, Type.INT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT2, Type.FLOAT4, Type.FLOAT4); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT2, Type.FLOAT8, Type.FLOAT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT2, Type.TEXT, Type.TEXT); - - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT4, Type.INT1, Type.INT4); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT4, Type.INT2, Type.INT4); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT4, Type.INT4, Type.INT4); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT4, Type.INT8, Type.INT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT4, Type.FLOAT4, Type.FLOAT4); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT4, Type.FLOAT8, Type.FLOAT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT4, Type.TEXT, Type.TEXT); - - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT8, Type.INT1, Type.INT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT8, Type.INT2, Type.INT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT8, Type.INT4, Type.INT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT8, Type.INT8, Type.INT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT8, Type.FLOAT4, Type.FLOAT4); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT8, Type.FLOAT8, Type.FLOAT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INT8, Type.TEXT, Type.TEXT); - - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.FLOAT4, Type.INT1, Type.FLOAT4); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.FLOAT4, Type.INT2, Type.FLOAT4); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.FLOAT4, Type.INT4, Type.FLOAT4); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.FLOAT4, Type.INT8, Type.FLOAT4); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.FLOAT4, Type.FLOAT4, Type.FLOAT4); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.FLOAT4, Type.FLOAT8, Type.FLOAT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.FLOAT4, Type.TEXT, Type.TEXT); - - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.FLOAT8, Type.INT1, Type.FLOAT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.FLOAT8, Type.INT2, Type.FLOAT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.FLOAT8, Type.INT4, Type.FLOAT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.FLOAT8, Type.INT8, Type.FLOAT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.FLOAT8, Type.FLOAT4, Type.FLOAT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.FLOAT8, Type.FLOAT8, Type.FLOAT8); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.FLOAT8, Type.TEXT, Type.TEXT); - - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.TEXT, Type.TIMESTAMP, Type.TIMESTAMP); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.TIMESTAMP, Type.TIMESTAMP, Type.TIMESTAMP); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.TIMESTAMP, Type.TEXT, Type.TEXT); - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.TEXT, Type.TEXT, Type.TEXT); - - TUtil.putToNestedMap(TYPE_CONVERSION_MAP, Type.INET4, Type.INET4, Type.INET4); - } } http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/planner/TypeDeterminant.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/TypeDeterminant.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/TypeDeterminant.java index 94a8d4a..511814c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/TypeDeterminant.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/TypeDeterminant.java @@ -27,7 +27,7 @@ import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.exception.NoSuchFunctionException; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.engine.utils.DataTypeUtil; +import org.apache.tajo.DataTypeUtil; import java.util.Stack; @@ -115,7 +115,7 @@ public class TypeDeterminant extends SimpleAlgebraVisitor<LogicalPlanner.PlanCon for (CaseWhenPredicate.WhenExpr when : caseWhen.getWhens()) { DataType resultType = visit(ctx, stack, when.getResult()); if (lastDataType != null) { - lastDataType = ExprAnnotator.getWidestType(lastDataType, resultType); + lastDataType = CatalogUtil.getWidestType(lastDataType, resultType); } else { lastDataType = resultType; } @@ -123,7 +123,7 @@ public class TypeDeterminant extends SimpleAlgebraVisitor<LogicalPlanner.PlanCon if (caseWhen.hasElseResult()) { DataType elseResultType = visit(ctx, stack, caseWhen.getElseResult()); - lastDataType = ExprAnnotator.getWidestType(lastDataType, elseResultType); + lastDataType = CatalogUtil.getWidestType(lastDataType, elseResultType); } return lastDataType; http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TruncateTableNode.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TruncateTableNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TruncateTableNode.java index b8d9cad..3ffbd7b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TruncateTableNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TruncateTableNode.java @@ -42,7 +42,7 @@ public class TruncateTableNode extends LogicalNode { @Override public String toString() { - return "TruncateTable (table=" + TUtil.collectionToString(tableNames) + ")"; + return "TruncateTable (table=" + TUtil.collectionToString(tableNames, ", ") + ")"; } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/GreedyHeuristicJoinOrderAlgorithm.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/GreedyHeuristicJoinOrderAlgorithm.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/GreedyHeuristicJoinOrderAlgorithm.java index f65fee7..e445de7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/GreedyHeuristicJoinOrderAlgorithm.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/GreedyHeuristicJoinOrderAlgorithm.java @@ -169,7 +169,7 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm { // If outer is outer join, make edge key using all relation names in outer. SortedSet<String> relationNames = new TreeSet<String>(PlannerUtil.getRelationLineageWithinQueryBlock(plan, outer)); - String outerEdgeKey = TUtil.collectionToString(relationNames); + String outerEdgeKey = TUtil.collectionToString(relationNames, ", "); for (String innerName : PlannerUtil.getRelationLineageWithinQueryBlock(plan, inner)) { if (graph.hasEdge(outerEdgeKey, innerName)) { JoinEdge existJoinEdge = graph.getEdge(outerEdgeKey, innerName); @@ -188,7 +188,7 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm { relationNames = new TreeSet<String>(PlannerUtil.getRelationLineageWithinQueryBlock(plan, inner)); - outerEdgeKey = TUtil.collectionToString(relationNames); + outerEdgeKey = TUtil.collectionToString(relationNames, ", "); for (String outerName : PlannerUtil.getRelationLineageWithinQueryBlock(plan, outer)) { if (graph.hasEdge(outerEdgeKey, outerName)) { JoinEdge existJoinEdge = graph.getEdge(outerEdgeKey, outerName); http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinEdge.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinEdge.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinEdge.java index c9bb571..eabdf13 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinEdge.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinEdge.java @@ -70,6 +70,6 @@ public class JoinEdge { } public String toString() { - return leftRelation + " " + joinType + " " + rightRelation + " ON " + TUtil.collectionToString(joinQual); + return leftRelation + " " + joinType + " " + rightRelation + " ON " + TUtil.collectionToString(joinQual, ", "); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java index 776e776..5a861f9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java @@ -110,7 +110,10 @@ public class JoinGraph extends SimpleUndirectedGraph<String, JoinEdge> { SortedSet<String> rightNodeRelationName = new TreeSet<String>(PlannerUtil.getRelationLineageWithinQueryBlock(plan, joinNode.getRightChild())); - addEdge(TUtil.collectionToString(leftNodeRelationName), TUtil.collectionToString(rightNodeRelationName), edge); + addEdge( + TUtil.collectionToString(leftNodeRelationName, ", "), + TUtil.collectionToString(rightNodeRelationName, ", "), + edge); Set<EvalNode> allInOneCnf = new HashSet<EvalNode>(); allInOneCnf.add(joinNode.getJoinQual()); http://git-wip-us.apache.org/repos/asf/tajo/blob/d56737b9/tajo-core/src/main/java/org/apache/tajo/engine/utils/DataTypeUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/DataTypeUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/DataTypeUtil.java deleted file mode 100644 index 084db21..0000000 --- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/DataTypeUtil.java +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.engine.utils; - -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.engine.eval.InvalidEvalException; - -public class DataTypeUtil { - /** - * This is verified by ExprsVerifier.checkArithmeticOperand(). - */ - public static TajoDataTypes.DataType determineType(TajoDataTypes.DataType left, TajoDataTypes.DataType right) throws InvalidEvalException { - switch (left.getType()) { - - case INT1: - case INT2: - case INT4: { - switch(right.getType()) { - case INT1: - case INT2: - case INT4: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT4); - case INT8: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT8); - case FLOAT4: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.FLOAT4); - case FLOAT8: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.FLOAT8); - case DATE: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.DATE); - case INTERVAL: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INTERVAL); - } - } - - case INT8: { - switch(right.getType()) { - case INT1: - case INT2: - case INT4: - case INT8: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT8); - case FLOAT4: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.FLOAT4); - case FLOAT8: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.FLOAT8); - case DATE: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.DATE); - case INTERVAL: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INTERVAL); - } - } - - case FLOAT4: { - switch(right.getType()) { - case INT1: - case INT2: - case INT4: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.FLOAT4); - case INT8: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.FLOAT4); - case FLOAT4: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.FLOAT4); - case FLOAT8: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.FLOAT8); - case INTERVAL: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INTERVAL); - } - } - - case FLOAT8: { - switch(right.getType()) { - case INT1: - case INT2: - case INT4: - case INT8: - case FLOAT4: - case FLOAT8: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.FLOAT8); - case INTERVAL: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INTERVAL); - } - } - - case DATE: { - switch(right.getType()) { - case INT2: - case INT4: - case INT8: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.DATE); - case INTERVAL: - case TIME: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TIMESTAMP); - case DATE: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT4); - } - } - - case TIME: { - switch(right.getType()) { - case INTERVAL: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TIME); - case TIME: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INTERVAL); - case DATE: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT4); - } - } - - case TIMESTAMP: { - switch (right.getType()) { - case INTERVAL: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TIMESTAMP); - case TIMESTAMP: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INTERVAL); - } - } - - case INTERVAL: { - switch (right.getType()) { - case INTERVAL: - case FLOAT4: - case FLOAT8: return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INTERVAL); - } - } - - default: return left; - } - } -}
