[
https://issues.apache.org/jira/browse/DRILL-3988?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16646797#comment-16646797
]
ASF GitHub Bot commented on DRILL-3988:
---------------------------------------
kkhatua closed pull request #1483: DRILL-3988: Expose Drill built-in functions
& UDFs in a system table
URL: https://github.com/apache/drill/pull/1483
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
index f4b83736739..a6f9a7f7ca7 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
@@ -28,6 +28,7 @@
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -54,6 +55,7 @@
import org.apache.drill.exec.exception.FunctionValidationException;
import org.apache.drill.exec.exception.JarValidationException;
import org.apache.drill.exec.expr.fn.registry.LocalFunctionRegistry;
+import org.apache.drill.exec.expr.fn.registry.FunctionHolder;
import org.apache.drill.exec.expr.fn.registry.JarScan;
import org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry;
import org.apache.drill.exec.planner.sql.DrillOperatorTable;
@@ -483,6 +485,17 @@ private ScanResult scan(ClassLoader classLoader, Path
path, URL[] urls) throws I
return missingJars;
}
+ /**
+ * Retrieve all functions, mapped by source jars (after syncing)
+ * @return Map of source jars and their functionHolders
+ */
+ public Map<String, List<FunctionHolder>> getAllJarsWithFunctionsHolders() {
+ if (useDynamicUdfs) {
+ syncWithRemoteRegistry(localFunctionRegistry.getVersion());
+ }
+ return localFunctionRegistry.getAllJarsWithFunctionsHolders();
+ }
+
/**
* Creates local udf directory, if it doesn't exist.
* Checks if local udf directory is a directory and if current application
has write rights on it.
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolder.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolder.java
index d1d4fc94dfd..d0383e91fc8 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolder.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolder.java
@@ -25,6 +25,8 @@
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
@@ -161,6 +163,38 @@ public void removeJar(String jarName) {
}
}
+ /**
+ * Retrieves all functions (holders) associated with all the jars
+ * This is read operation, so several users can perform this operation at
the same time.
+ * @return list of all functions, mapped by their sources
+ */
+ public Map<String, List<FunctionHolder>> getAllJarsWithFunctionHolders() {
+ Map<String, List<FunctionHolder>> allFunctionHoldersByJar = new
HashMap<>();
+
+ try (@SuppressWarnings("unused") Closeable lock = readLock.open()) {
+ for (String jarName : jars.keySet()) {
+ //Capture functionHolders here
+ List<FunctionHolder> drillFuncHolderList = new LinkedList<>();
+
+ Map<String, Queue<String>> functionsInJar = jars.get(jarName);
+ for (Map.Entry<String, Queue<String>> functionEntry :
functionsInJar.entrySet()) {
+ String fnName = functionEntry.getKey();
+ Queue<String> fnSignatureList = functionEntry.getValue();
+ //Get all FunctionHolders (irrespective of source)
+ Map<String, DrillFuncHolder> functionHolders = functions.get(fnName);
+ //Iterate for matching entries and populate new Map
+ for (Map.Entry<String, DrillFuncHolder> entry :
functionHolders.entrySet()) {
+ if (fnSignatureList.contains(entry.getKey())) {
+ drillFuncHolderList.add(new FunctionHolder(fnName,
entry.getKey(), entry.getValue()));
+ }
+ }
+ }
+ allFunctionHoldersByJar.put(jarName, drillFuncHolderList);
+ }
+ }
+ return allFunctionHoldersByJar;
+ }
+
/**
* Retrieves all function names associated with the jar from {@link #jars}.
* Returns empty list if jar is not registered.
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/LocalFunctionRegistry.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/LocalFunctionRegistry.java
index cefbd8cf388..f96b1fbb6ed 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/LocalFunctionRegistry.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/LocalFunctionRegistry.java
@@ -235,6 +235,14 @@ public int size(){
return registryHolder.getHoldersByFunctionName(name.toLowerCase());
}
+ /**
+ * Returns a map of all function holders mapped by source jars
+ * @return all functions organized by source jars
+ */
+ public Map<String, List<FunctionHolder>> getAllJarsWithFunctionsHolders() {
+ return registryHolder.getAllJarsWithFunctionHolders();
+ }
+
/**
* Registers all functions present in {@link DrillOperatorTable},
* also sets sync registry version used at the moment of function
registration.
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java
index a8980785a32..cfb242a3650 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java
@@ -239,6 +239,8 @@ public void setBuffers(final IncomingBuffers buffers) {
Preconditions.checkArgument(this.buffers == null, "Can only set buffers
once.");
this.buffers = buffers;
}
+
+ @Override
public QueryProfileStoreContext getProfileStoreContext() {
return context.getProfileStoreContext();
}
@@ -248,6 +250,7 @@ public QueryProfileStoreContext getProfileStoreContext() {
return context.getUserConnections();
}
+ @Override
public void setExecutorState(final ExecutorState executorState) {
Preconditions.checkArgument(this.executorState == null, "ExecutorState can
only be set once.");
this.executorState = executorState;
@@ -257,6 +260,7 @@ public void fail(final Throwable cause) {
executorState.fail(cause);
}
+ @Override
public SchemaPlus getFullRootSchema() {
if (queryContext == null) {
fail(new UnsupportedOperationException("Schema tree can only be created
in root fragment. " +
@@ -278,6 +282,7 @@ public SchemaPlus getFullRootSchema() {
return queryContext.getFullRootSchema(schemaConfig);
}
+ @Override
public FragmentStats getStats() {
return stats;
}
@@ -326,10 +331,12 @@ public ExecutorService getScanExecutor() {
* The FragmentHandle for this Fragment
* @return FragmentHandle
*/
+ @Override
public FragmentHandle getHandle() {
return fragment.getHandle();
}
+ @Override
public String getFragIdString() {
final FragmentHandle handle = getHandle();
final String frag = handle != null ? handle.getMajorFragmentId() + ":" +
handle.getMinorFragmentId() : "0:0";
@@ -361,6 +368,7 @@ public RuntimeFilterWritable getRuntimeFilter() {
* Get this fragment's allocator.
* @return the allocator
*/
+ @Override
@Deprecated
public BufferAllocator getAllocator() {
if (allocator == null) {
@@ -411,10 +419,12 @@ public AccountingDataTunnel getDataTunnel(final
DrillbitEndpoint endpoint) {
return tunnel;
}
+ @Override
public IncomingBuffers getBuffers() {
return buffers;
}
+ @Override
public OperatorContext newOperatorContext(PhysicalOperator popConfig,
OperatorStats stats)
throws OutOfMemoryException {
OperatorContextImpl context = new OperatorContextImpl(popConfig, this,
stats);
@@ -422,6 +432,7 @@ public OperatorContext newOperatorContext(PhysicalOperator
popConfig, OperatorSt
return context;
}
+ @Override
public OperatorContext newOperatorContext(PhysicalOperator popConfig)
throws OutOfMemoryException {
OperatorContextImpl context = new OperatorContextImpl(popConfig, this);
@@ -444,10 +455,12 @@ public ExecutionControls getExecutionControls() {
return executionControls;
}
+ @Override
public String getQueryUserName() {
return fragment.getCredentials().getUserName();
}
+ @Override
public boolean isImpersonationEnabled() {
// TODO(DRILL-2097): Until SimpleRootExec tests are removed, we need to
consider impersonation disabled if there is
// no config
@@ -509,6 +522,7 @@ public ValueHolder getConstantValueHolder(String value,
MinorType type, Function
return valueHolder;
}
+ @Override
public ExecutorService getExecutor(){
return context.getExecutor();
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
index b047f7d8b6d..b2802de7118 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
@@ -261,7 +261,9 @@ public ScanResult getClasspathScan() {
return classpathScan;
}
- public RemoteFunctionRegistry getRemoteFunctionRegistry() { return
functionRegistry.getRemoteFunctionRegistry(); }
+ public RemoteFunctionRegistry getRemoteFunctionRegistry() {
+ return functionRegistry.getRemoteFunctionRegistry();
+ }
/**
* Use the operator table built during startup when "exec.udf.use_dynamic"
option
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/FunctionsIterator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/FunctionsIterator.java
new file mode 100644
index 00000000000..86e88174e8d
--- /dev/null
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/FunctionsIterator.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.drill.exec.expr.fn.DrillFuncHolder;
+import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.expr.fn.FunctionLookupContext;
+import org.apache.drill.exec.expr.fn.registry.FunctionHolder;
+import org.apache.drill.exec.ops.ExecutorFragmentContext;
+import org.apache.drill.exec.store.pojo.NonNullable;
+
+/**
+ * List functions as a System Table
+ */
+public class FunctionsIterator implements Iterator<Object> {
+ private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(FunctionsIterator.class);
+
+ private final Iterator<FunctionInfo> sortedIterator;
+
+ public FunctionsIterator(ExecutorFragmentContext context) {
+ Map<String, FunctionInfo> functionMap = new HashMap<String,
FunctionInfo>();
+ //Access Registry for function list
+ FunctionLookupContext functionLookupContext =
context.getFunctionRegistry();
+ //Check true instance type
+ if (functionLookupContext instanceof FunctionImplementationRegistry) {
+ @SuppressWarnings("resource")
+ FunctionImplementationRegistry functionImplRegistry =
(FunctionImplementationRegistry) functionLookupContext;
+ Map<String, List<FunctionHolder>> jarFunctionListMap =
functionImplRegistry.getAllJarsWithFunctionsHolders();
+
+ for (String jarName : jarFunctionListMap.keySet()) {
+ for (FunctionHolder dfhEntry : jarFunctionListMap.get(jarName)) {
+ populateFunctionMap(functionMap, jarName, dfhEntry.getHolder());
+ }
+ }
+
+ List<FunctionInfo> functionList = new
ArrayList<FunctionsIterator.FunctionInfo>(functionMap.values());
+ functionList.sort((FunctionInfo o1, FunctionInfo o2) -> {
+ int result = o1.name.compareTo(o2.name);
+ if (result == 0) {
+ result = o1.signature.compareTo(o2.signature);
+ }
+ if (result == 0) {
+ return o1.returnType.compareTo(o2.returnType);
+ }
+ return result;
+ });
+
+ sortedIterator = functionList.iterator();
+ } else {
+ //Logging error
+ logger.error("Function Registry was found to be of {} instead of {}. No
functions could be loaded ",
+ FunctionImplementationRegistry.class,
functionLookupContext.getClass());
+ sortedIterator = Collections.emptyIterator();
+ }
+
+ }
+
+ /**
+ * Populate the map of functionInfo based on the functionSignatureKey for a
given Jar-DrillFunctionHolder
+ * @param functionMap map to populate
+ * @param jarName name of the source jar
+ * @param dfh functionHolder that carries all the registered names and the
signature
+ */
+ private void populateFunctionMap(Map<String, FunctionInfo> functionMap,
String jarName, DrillFuncHolder dfh) {
+ String registeredNames[] = dfh.getRegisteredNames();
+ String signature = dfh.getInputParameters();
+ String returnType = dfh.getReturnType().getMinorType().toString();
+ for (String name : registeredNames) {
+ //Generate a unique key for a function holder as
'functionName#functionSignature'
+ //Bumping capacity from default 16 to 64 (since key is expected to be
bigger, and reduce probability of resizing)
+ String funcSignatureKey = new
StringBuilder(64).append(name).append('#').append(signature).toString();
+ functionMap.put(funcSignatureKey, new FunctionInfo(name, signature,
returnType, jarName));
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return sortedIterator.hasNext();
+ }
+
+ @Override
+ public FunctionInfo next() {
+ return sortedIterator.next();
+ }
+
+ /**
+ * Representation of an entry in the System table - Functions
+ */
+ public static class FunctionInfo {
+ @NonNullable
+ public final String name;
+ @NonNullable
+ public final String signature;
+ @NonNullable
+ public final String returnType;
+ @NonNullable
+ public final String source;
+
+ public FunctionInfo(String funcName, String funcSignature, String
funcReturnType, String jarName) {
+ this.name = funcName;
+ this.signature = funcSignature;
+ this.returnType = funcReturnType;
+ this.source = jarName;
+ }
+ }
+}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
index 62e610d73fc..fae43522188 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
@@ -112,9 +112,16 @@
THREADS("threads", true, ThreadsIterator.ThreadsInfo.class) {
@Override
- public Iterator<Object> getIterator(final ExecutorFragmentContext context,
final int maxRecords) {
+ public Iterator<Object> getIterator(final ExecutorFragmentContext context,
final int maxRecords) {
return new ThreadsIterator(context);
}
+ },
+
+ FUNCTIONS("functions", false, FunctionsIterator.FunctionInfo.class) {
+ @Override
+ public Iterator<Object> getIterator(final ExecutorFragmentContext context,
final int maxRecords) {
+ return new FunctionsIterator(context);
+ }
};
private final String tableName;
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolderTest.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolderTest.java
index 978231575ae..3e8e6b87e09 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolderTest.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolderTest.java
@@ -19,7 +19,6 @@
import
org.apache.drill.shaded.guava.com.google.common.collect.ArrayListMultimap;
import org.apache.drill.shaded.guava.com.google.common.collect.ListMultimap;
-import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
import org.junit.Before;
@@ -28,10 +27,14 @@
import org.junit.experimental.categories.Category;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@@ -46,6 +49,8 @@
private static final String built_in = "built-in";
private static final String udf_jar = "DrillUDF-1.0.jar";
+ private static final String LOWER_FUNC_NAME = "lower";
+ private static final String SHUFFLE_FUNC_NAME = "shuffle";
private static Map<String, List<FunctionHolder>> newJars;
private FunctionRegistryHolder registryHolder;
@@ -53,12 +58,14 @@
@BeforeClass
public static void init() {
newJars = new HashMap<>();
- FunctionHolder lower = new FunctionHolder("lower",
"lower(VARCHAR-REQUIRED)", mock(DrillFuncHolder.class));
+ FunctionHolder lower = new FunctionHolder(LOWER_FUNC_NAME,
"lower(VARCHAR-REQUIRED)", mock(DrillFuncHolder.class));
FunctionHolder upper = new FunctionHolder("upper",
"upper(VARCHAR-REQUIRED)", mock(DrillFuncHolder.class));
- newJars.put(built_in, Lists.newArrayList(lower, upper));
+ FunctionHolder shuffle = new FunctionHolder(SHUFFLE_FUNC_NAME,
"shuffle()", mock(DrillFuncHolder.class));
+ newJars.put(built_in, new ArrayList<>(Arrays.asList(lower, upper,
shuffle)));
FunctionHolder custom_lower = new FunctionHolder("custom_lower",
"lower(VARCHAR-REQUIRED)", mock(DrillFuncHolder.class));
FunctionHolder custom_upper = new FunctionHolder("custom_upper",
"custom_upper(VARCHAR-REQUIRED)", mock(DrillFuncHolder.class));
- newJars.put(udf_jar, Lists.newArrayList(custom_lower, custom_upper));
+ FunctionHolder overloaded_shuffle = new FunctionHolder(SHUFFLE_FUNC_NAME,
"shuffle(FLOAT8-REQUIRED,FLOAT8-OPTIONAL)", mock(DrillFuncHolder.class));
+ newJars.put(udf_jar, new ArrayList<>(Arrays.asList(custom_lower,
custom_upper, overloaded_shuffle)));
}
@Before
@@ -87,16 +94,16 @@ public void testVersion() {
@Test
public void testAddJars() {
resetRegistry();
- int functionsSize = 0;
List<String> jars = new ArrayList<>();
ListMultimap<String, DrillFuncHolder> functionsWithHolders =
ArrayListMultimap.create();
ListMultimap<String, String> functionsWithSignatures =
ArrayListMultimap.create();
+ Set<String> functionsSet = new HashSet<>();
for (Map.Entry<String, List<FunctionHolder>> jar : newJars.entrySet()) {
jars.add(jar.getKey());
for (FunctionHolder functionHolder : jar.getValue()) {
functionsWithHolders.put(functionHolder.getName(),
functionHolder.getHolder());
functionsWithSignatures.put(functionHolder.getName(),
functionHolder.getSignature());
- functionsSize++;
+ functionsSet.add(functionHolder.getName()); //Track unique function
names
}
}
@@ -104,7 +111,7 @@ public void testAddJars() {
registryHolder.addJars(newJars, ++expectedVersion);
assertEquals("Version number should match", expectedVersion,
registryHolder.getVersion());
compareTwoLists(jars, registryHolder.getAllJarNames());
- assertEquals(functionsSize, registryHolder.functionsSize());
+ assertEquals(functionsSet.size(), registryHolder.functionsSize());
compareListMultimaps(functionsWithHolders,
registryHolder.getAllFunctionsWithHolders());
compareListMultimaps(functionsWithSignatures,
registryHolder.getAllFunctionsWithSignatures());
}
@@ -112,7 +119,7 @@ public void testAddJars() {
@Test
public void testAddTheSameJars() {
resetRegistry();
- int functionsSize = 0;
+ Set<String> functionsSet = new HashSet<>();
List<String> jars = new ArrayList<>();
ListMultimap<String, DrillFuncHolder> functionsWithHolders =
ArrayListMultimap.create();
ListMultimap<String, String> functionsWithSignatures =
ArrayListMultimap.create();
@@ -121,14 +128,14 @@ public void testAddTheSameJars() {
for (FunctionHolder functionHolder : jar.getValue()) {
functionsWithHolders.put(functionHolder.getName(),
functionHolder.getHolder());
functionsWithSignatures.put(functionHolder.getName(),
functionHolder.getSignature());
- functionsSize++;
+ functionsSet.add(functionHolder.getName()); //Track unique function
names
}
}
int expectedVersion = 0;
registryHolder.addJars(newJars, ++expectedVersion);
assertEquals("Version number should match", expectedVersion,
registryHolder.getVersion());
compareTwoLists(jars, registryHolder.getAllJarNames());
- assertEquals(functionsSize, registryHolder.functionsSize());
+ assertEquals(functionsSet.size(), registryHolder.functionsSize());
compareListMultimaps(functionsWithHolders,
registryHolder.getAllFunctionsWithHolders());
compareListMultimaps(functionsWithSignatures,
registryHolder.getAllFunctionsWithSignatures());
@@ -136,7 +143,7 @@ public void testAddTheSameJars() {
registryHolder.addJars(newJars, ++expectedVersion);
assertEquals("Version number should match", expectedVersion,
registryHolder.getVersion());
compareTwoLists(jars, registryHolder.getAllJarNames());
- assertEquals(functionsSize, registryHolder.functionsSize());
+ assertEquals(functionsSet.size(), registryHolder.functionsSize());
compareListMultimaps(functionsWithHolders,
registryHolder.getAllFunctionsWithHolders());
compareListMultimaps(functionsWithSignatures,
registryHolder.getAllFunctionsWithSignatures());
}
@@ -155,6 +162,47 @@ public void testGetAllJarNames() {
compareTwoLists(expectedResult, registryHolder.getAllJarNames());
}
+ @Test
+ public void testGetAllJarsWithFunctionHolders() {
+ Map<String, List<FunctionHolder>> fnHoldersInRegistry =
registryHolder.getAllJarsWithFunctionHolders();
+ //Iterate and confirm lists are same
+ for (String jarName : newJars.keySet()) {
+ List<DrillFuncHolder> expectedHolderList = newJars.get(jarName).stream()
+ .map(FunctionHolder::getHolder) //Extract DrillFuncHolder
+ .collect(Collectors.toList());
+ List<DrillFuncHolder> testHolderList =
fnHoldersInRegistry.get(jarName).stream()
+ .map(FunctionHolder::getHolder) //Extract DrillFuncHolder
+ .collect(Collectors.toList());
+
+ compareTwoLists(expectedHolderList, testHolderList);
+ }
+
+ Map<String, String> shuffleFunctionMap = new HashMap<>();
+ // Confirm that same function spans multiple jars with different signatures
+ //Init: Expected Map of items
+ for (String jarName : newJars.keySet()) {
+ for (FunctionHolder funcHolder : newJars.get(jarName)) {
+ if (SHUFFLE_FUNC_NAME.equals(funcHolder.getName())) {
+ shuffleFunctionMap.put(funcHolder.getSignature(), jarName);
+ }
+ }
+ }
+
+ //Test: Remove items from ExpectedMap based on match from testJar's
functionHolder items
+ for (String testJar : registryHolder.getAllJarNames()) {
+ for (FunctionHolder funcHolder : fnHoldersInRegistry.get(testJar)) {
+ if (SHUFFLE_FUNC_NAME.equals(funcHolder.getName())) {
+ String testSignature = funcHolder.getSignature();
+ String expectedJar = shuffleFunctionMap.get(testSignature);
+ if (testJar.equals(expectedJar)) {
+ shuffleFunctionMap.remove(testSignature);
+ }
+ }
+ }
+ }
+ assertTrue(shuffleFunctionMap.isEmpty());
+ }
+
@Test
public void testGetFunctionNamesByJar() {
List<String> expectedResult = newJars.get(built_in).stream()
@@ -202,26 +250,38 @@ public void testGetAllFunctionsWithSignatures() {
public void testGetHoldersByFunctionNameWithVersion() {
List<DrillFuncHolder> expectedResult = newJars.values().stream()
.flatMap(Collection::stream)
- .filter(f -> "lower".equals(f.getName()))
+ .filter(f -> LOWER_FUNC_NAME.equals(f.getName()))
.map(FunctionHolder::getHolder)
.collect(Collectors.toList());
assertFalse(expectedResult.isEmpty());
AtomicInteger version = new AtomicInteger();
- compareTwoLists(expectedResult,
registryHolder.getHoldersByFunctionName("lower", version));
+ compareTwoLists(expectedResult,
registryHolder.getHoldersByFunctionName(LOWER_FUNC_NAME, version));
assertEquals("Version number should match", version.get(),
registryHolder.getVersion());
}
@Test
public void testGetHoldersByFunctionName() {
- List<DrillFuncHolder> expectedResult = newJars.values().stream()
- .flatMap(Collection::stream)
- .filter(f -> "lower".equals(f.getName()))
- .map(FunctionHolder::getHolder)
- .collect(Collectors.toList());
+ List<DrillFuncHolder> expectedUniqueResult = new ArrayList<>();
+ List<DrillFuncHolder> expectedMultipleResult = new ArrayList<>();
+ for (List<FunctionHolder> functionHolders : newJars.values()) {
+ for (FunctionHolder functionHolder : functionHolders) {
+ if (LOWER_FUNC_NAME.equals(functionHolder.getName())) {
+ expectedUniqueResult.add(functionHolder.getHolder());
+ } else
+ if (SHUFFLE_FUNC_NAME.equals(functionHolder.getName())) {
+ expectedMultipleResult.add(functionHolder.getHolder());
+ }
+ }
+ }
- assertFalse(expectedResult.isEmpty());
- compareTwoLists(expectedResult,
registryHolder.getHoldersByFunctionName("lower"));
+ //Test for function with one signature
+ assertFalse(expectedUniqueResult.isEmpty());
+ compareTwoLists(expectedUniqueResult,
registryHolder.getHoldersByFunctionName(LOWER_FUNC_NAME));
+
+ //Test for function with multiple signatures
+ assertFalse(expectedMultipleResult.isEmpty());
+ compareTwoLists(expectedMultipleResult,
registryHolder.getHoldersByFunctionName(SHUFFLE_FUNC_NAME));
}
@Test
@@ -232,10 +292,23 @@ public void testContainsJar() {
@Test
public void testFunctionsSize() {
- int count = newJars.values().stream()
- .mapToInt(List::size)
- .sum();
- assertEquals("Functions size should match", count,
registryHolder.functionsSize());
+ int fnCountInRegistryHolder = 0;
+ int fnCountInNewJars = 0;
+
+ Set<String> functionNameSet = new HashSet<>();
+ for (List<FunctionHolder> functionHolders : newJars.values()) {
+ for (FunctionHolder functionHolder : functionHolders) {
+ functionNameSet.add(functionHolder.getName()); //Track unique function
names
+ fnCountInNewJars++; //Track all functions
+ }
+ }
+ assertEquals("Unique function name count should match",
functionNameSet.size(), registryHolder.functionsSize());
+
+ for (String jarName : registryHolder.getAllJarNames()) {
+ fnCountInRegistryHolder +=
registryHolder.getFunctionNamesByJar(jarName).size();
+ }
+
+ assertEquals("Function count should match", fnCountInNewJars,
fnCountInRegistryHolder);
}
@Test
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java
index 4f47287aac1..5c9990a4b63 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java
@@ -76,6 +76,11 @@ public void connectionsTable() throws Exception {
test("select * from sys.connections");
}
+ @Test
+ public void functionsTable() throws Exception {
+ test("select * from sys.functions");
+ }
+
@Test
public void profilesTable() throws Exception {
test("select * from sys.profiles");
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
index de55f19e59f..ce58cb05af3 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
@@ -149,7 +149,7 @@ public void tables() throws Exception {
assertEquals(RequestStatus.OK, resp.getStatus());
List<TableMetadata> tables = resp.getTablesList();
- assertEquals(18, tables.size());
+ assertEquals(19, tables.size());
verifyTable("information_schema", "CATALOGS", tables);
verifyTable("information_schema", "COLUMNS", tables);
@@ -157,15 +157,10 @@ public void tables() throws Exception {
verifyTable("information_schema", "TABLES", tables);
verifyTable("information_schema", "VIEWS", tables);
verifyTable("information_schema", "FILES", tables);
- verifyTable("sys", "boot", tables);
- verifyTable("sys", "drillbits", tables);
- verifyTable("sys", "memory", tables);
- verifyTable("sys", SystemTable.OPTIONS_OLD.getTableName(), tables);
- verifyTable("sys", SystemTable.OPTIONS.getTableName(), tables);
- verifyTable("sys", "threads", tables);
- verifyTable("sys", "version", tables);
- verifyTable("sys", SystemTable.INTERNAL_OPTIONS_OLD.getTableName(),
tables);
- verifyTable("sys", SystemTable.INTERNAL_OPTIONS.getTableName(), tables);
+ //Verify System Tables
+ for (SystemTable sysTbl : SystemTable.values()) {
+ verifyTable("sys", sysTbl.getTableName(), tables);
+ }
}
@Test
@@ -187,7 +182,7 @@ public void tablesWithSystemTableFilter() throws Exception {
assertEquals(RequestStatus.OK, resp.getStatus());
List<TableMetadata> tables = resp.getTablesList();
- assertEquals(18, tables.size());
+ assertEquals(19, tables.size());
verifyTable("information_schema", "CATALOGS", tables);
verifyTable("information_schema", "COLUMNS", tables);
@@ -195,15 +190,10 @@ public void tablesWithSystemTableFilter() throws
Exception {
verifyTable("information_schema", "TABLES", tables);
verifyTable("information_schema", "VIEWS", tables);
verifyTable("information_schema", "FILES", tables);
- verifyTable("sys", "boot", tables);
- verifyTable("sys", "drillbits", tables);
- verifyTable("sys", "memory", tables);
- verifyTable("sys", SystemTable.OPTIONS_OLD.getTableName(), tables);
- verifyTable("sys", SystemTable.OPTIONS.getTableName(), tables);
- verifyTable("sys", "threads", tables);
- verifyTable("sys", "version", tables);
- verifyTable("sys", SystemTable.INTERNAL_OPTIONS_OLD.getTableName(),
tables);
- verifyTable("sys", SystemTable.INTERNAL_OPTIONS.getTableName(), tables);
+ //Verify System Tables
+ for (SystemTable sysTbl : SystemTable.values()) {
+ verifyTable("sys", sysTbl.getTableName(), tables);
+ }
}
@Test
@@ -216,15 +206,15 @@ public void tablesWithTableNameFilter() throws Exception {
assertEquals(RequestStatus.OK, resp.getStatus());
List<TableMetadata> tables = resp.getTablesList();
- assertEquals(10, tables.size());
-
- verifyTable("sys", "boot", tables);
- verifyTable("sys", "memory", tables);
- verifyTable("sys", SystemTable.OPTIONS_OLD.getTableName(), tables);
- verifyTable("sys", SystemTable.OPTIONS.getTableName(), tables);
- verifyTable("sys", "version", tables);
- verifyTable("sys", SystemTable.INTERNAL_OPTIONS_OLD.getTableName(),
tables);
- verifyTable("sys", SystemTable.INTERNAL_OPTIONS.getTableName(), tables);
+ assertEquals(11, tables.size());
+
+ //Verify System Tables
+ for (SystemTable sysTbl : SystemTable.values()) {
+ String sysTblName = sysTbl.getTableName();
+ if (sysTblName.contains("o")) {
+ verifyTable("sys", sysTblName, tables);
+ }
+ }
}
@Test
@@ -250,7 +240,7 @@ public void columns() throws Exception {
assertEquals(RequestStatus.OK, resp.getStatus());
List<ColumnMetadata> columns = resp.getColumnsList();
- assertEquals(135, columns.size());
+ assertEquals(139, columns.size());
// too many records to verify the output.
}
@@ -265,11 +255,11 @@ public void columnsWithColumnNameFilter() throws
Exception {
List<ColumnMetadata> columns = resp.getColumnsList();
assertEquals(6, columns.size());
- verifyColumn("sys", "drillbits", "user_port", columns);
- verifyColumn("sys", "drillbits", "control_port", columns);
- verifyColumn("sys", "drillbits", "data_port", columns);
- verifyColumn("sys", "memory", "user_port", columns);
- verifyColumn("sys", "threads", "user_port", columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "user_port",
columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "control_port",
columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "data_port",
columns);
+ verifyColumn("sys", SystemTable.MEMORY.getTableName(), "user_port",
columns);
+ verifyColumn("sys", SystemTable.THREADS.getTableName(), "user_port",
columns);
}
@Test
@@ -285,9 +275,9 @@ public void columnsWithColumnNameFilterAndTableNameFilter()
throws Exception {
List<ColumnMetadata> columns = resp.getColumnsList();
assertEquals(4, columns.size());
- verifyColumn("sys", "drillbits", "user_port", columns);
- verifyColumn("sys", "drillbits", "control_port", columns);
- verifyColumn("sys", "drillbits", "data_port", columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "user_port",
columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "control_port",
columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "data_port",
columns);
}
@Test
@@ -306,10 +296,10 @@ public void columnsWithAllSupportedFilters() throws
Exception {
List<ColumnMetadata> columns = resp.getColumnsList();
assertEquals(4, columns.size());
- verifyColumn("sys", "drillbits", "user_port", columns);
- verifyColumn("sys", "drillbits", "control_port", columns);
- verifyColumn("sys", "drillbits", "data_port", columns);
- verifyColumn("sys", "drillbits", "http_port", columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "user_port",
columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "control_port",
columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "data_port",
columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "http_port",
columns);
}
/** Helper method to verify schema contents */
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> Create a sys.functions table to expose available Drill functions
> ----------------------------------------------------------------
>
> Key: DRILL-3988
> URL: https://issues.apache.org/jira/browse/DRILL-3988
> Project: Apache Drill
> Issue Type: Sub-task
> Components: Metadata
> Reporter: Jacques Nadeau
> Assignee: Kunal Khatua
> Priority: Major
> Labels: doc-impacting
> Fix For: 1.15.0
>
>
> Create a new sys.functions table that returns a list of all available
> functions.
> Key considerations:
> - one row per name or one per argument set. I'm inclined to latter so people
> can use queries to get to data.
> - we need to create a delineation between user functions and internal
> functions and only show user functions. 'CastInt' isn't something the user
> should be able to see (or run).
> - should we add a description annotation that could be included in the
> sys.functions table?
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
