This is an automated email from the ASF dual-hosted git repository.
kunal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new a26d848 DRILL-6084: Show Drill functions in WebUI for autocomplete
a26d848 is described below
commit a26d848643e8d7851d8651a4d6c1f366624c3a70
Author: Kunal Khatua <[email protected]>
AuthorDate: Wed Oct 17 17:30:03 2018 -0700
DRILL-6084: Show Drill functions in WebUI for autocomplete
Building on top of DRILL-3988 and leveraging DRILL-5868, this allows
support for Drill functions to be now available in the WebUI.
If users wants UDFs to show up, they should place the UDF jars in the
`$DRILL_HOME/jars/3rdparty` directory so that this can be loaded during the
Drillbit's startup.
Concept of internal Drill functions are introduced. With this, internal
Drill functions like `ConvertToNullableXYZ` has been marked as internal.
The WebUI will not show these functions. However, they are still visible in
`sys.functions` table with an additional column indicating that it is an
internal function.
Tests have been added as a part of this commit to verify the internal
functions concept.
---
.../CastEmptyStringVarTypesToNullableNumeric.java | 2 +-
.../codegen/templates/ConvertToNullableHolder.java | 3 +-
.../exec/expr/annotations/FunctionTemplate.java | 5 ++
.../apache/drill/exec/expr/fn/DrillFuncHolder.java | 4 +
.../drill/exec/expr/fn/FunctionAttributes.java | 4 +
.../expr/fn/FunctionImplementationRegistry.java | 4 +
.../apache/drill/exec/expr/fn/ValueReference.java | 9 +++
.../apache/drill/exec/server/rest/WebServer.java | 86 +++++++++++++++++++---
.../drill/exec/store/sys/FunctionsIterator.java | 10 ++-
.../mode-sql.js => ace.mode-sql.template.js} | 17 +----
.../src/main/resources/rest/query/query.ftl | 3 +-
.../drill/exec/store/sys/TestSystemTable.java | 7 ++
.../exec/work/metadata/TestMetadataProvider.java | 2 +-
13 files changed, 125 insertions(+), 31 deletions(-)
diff --git
a/exec/java-exec/src/main/codegen/templates/CastEmptyStringVarTypesToNullableNumeric.java
b/exec/java-exec/src/main/codegen/templates/CastEmptyStringVarTypesToNullableNumeric.java
index e0942e9..6512771 100644
---
a/exec/java-exec/src/main/codegen/templates/CastEmptyStringVarTypesToNullableNumeric.java
+++
b/exec/java-exec/src/main/codegen/templates/CastEmptyStringVarTypesToNullableNumeric.java
@@ -48,7 +48,7 @@
*/
@SuppressWarnings("unused")
-@FunctionTemplate(name = "castEmptyString${type.from}To${type.to?upper_case}",
scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL)
+@FunctionTemplate(name = "castEmptyString${type.from}To${type.to?upper_case}",
scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL,
isInternal=true)
public class CastEmptyString${type.from}To${type.to} implements
DrillSimpleFunc{
@Param ${type.from}Holder in;
diff --git
a/exec/java-exec/src/main/codegen/templates/ConvertToNullableHolder.java
b/exec/java-exec/src/main/codegen/templates/ConvertToNullableHolder.java
index 7615bc5..875038c 100644
--- a/exec/java-exec/src/main/codegen/templates/ConvertToNullableHolder.java
+++ b/exec/java-exec/src/main/codegen/templates/ConvertToNullableHolder.java
@@ -42,7 +42,8 @@ import org.apache.drill.exec.record.RecordBatch;
<#elseif minor.class.startsWith("Var")>
returnType = FunctionTemplate.ReturnType.SAME_IN_OUT_LENGTH,
</#if>
- nulls = FunctionTemplate.NullHandling.INTERNAL)
+ nulls = FunctionTemplate.NullHandling.INTERNAL,
+ isInternal = true)
public class ${className} implements DrillSimpleFunc {
@Param ${minor.class}Holder input;
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java
index 11914ea..3aa7365 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java
@@ -91,6 +91,11 @@ public @interface FunctionTemplate {
boolean checkPrecisionRange() default false;
/**
+ * Defines if a function is internal and not intended for public use [e.g.
castEmptyStringNullableVarBinaryToNullableVarDecimal(..) ]
+ */
+ boolean isInternal() default false;
+
+ /**
* This enum will be used to estimate the average size of the output
* produced by a function that produces variable length output
*/
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
index d6f6ea2..fc021c9 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
@@ -107,6 +107,10 @@ public abstract class DrillFuncHolder extends
AbstractFuncHolder {
}
+ public boolean isInternal() {
+ return attributes.isInternal();
+ }
+
/**
* Generates string representation of function input parameters:
* PARAMETER_TYPE_1-PARAMETER_MODE_1,PARAMETER_TYPE_2-PARAMETER_MODE_2
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionAttributes.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionAttributes.java
index 6d1b767..5f744e5 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionAttributes.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionAttributes.java
@@ -103,5 +103,9 @@ public class FunctionAttributes {
return template.isNiladic();
}
+ public boolean isInternal() {
+ return template.isInternal();
+ }
+
public boolean checkPrecisionRange() { return
template.checkPrecisionRange(); }
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
index a6f9a7f..5621c44 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
@@ -293,6 +293,10 @@ public class FunctionImplementationRegistry implements
FunctionLookupContext, Au
return false;
}
+ public LocalFunctionRegistry getLocalFunctionRegistry() {
+ return localFunctionRegistry;
+ }
+
public RemoteFunctionRegistry getRemoteFunctionRegistry() {
return remoteFunctionRegistry;
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ValueReference.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ValueReference.java
index 42d3967..1ee39ff 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ValueReference.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ValueReference.java
@@ -28,6 +28,7 @@ public class ValueReference {
private boolean isConstant = false;
private boolean isFieldReader = false;
private boolean isComplexWriter = false;
+ private boolean isInternal = false;
public ValueReference(MajorType type, String name) {
Preconditions.checkNotNull(type);
@@ -52,6 +53,14 @@ public class ValueReference {
return isConstant;
}
+ public void setInternal(boolean isInternal) {
+ this.isInternal = isInternal;
+ }
+
+ public boolean isInternal() {
+ return isInternal;
+ }
+
public boolean isFieldReader() {
return isFieldReader;
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java
index fd9950e..8cab4dd 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java
@@ -20,8 +20,6 @@ package org.apache.drill.exec.server.rest;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.servlets.MetricsServlet;
import com.codahale.metrics.servlets.ThreadDumpServlet;
-import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
-import org.apache.drill.shaded.guava.com.google.common.io.Files;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
@@ -32,6 +30,8 @@ import org.apache.drill.common.exceptions.DrillException;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ssl.SSLConfig;
import org.apache.drill.exec.exception.DrillbitStartupException;
+import org.apache.drill.exec.expr.fn.registry.FunctionHolder;
+import org.apache.drill.exec.expr.fn.registry.LocalFunctionRegistry;
import org.apache.drill.exec.server.BootStrapContext;
import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.options.OptionList;
@@ -84,20 +84,34 @@ import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.net.BindException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
import java.security.KeyStore;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.EnumSet;
import java.util.List;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
/**
* Wrapper class around jetty based webserver.
*/
public class WebServer implements AutoCloseable {
+ private static final String ACE_MODE_SQL_TEMPLATE_JS =
"ace.mode-sql.template.js";
+ private static final String ACE_MODE_SQL_JS = "mode-sql.js";
+ private static final String DRILL_FUNCTIONS_PLACEHOLDER =
"__DRILL_FUNCTIONS__";
+
+ private static final String STATUS_THREADS_PATH = "/status/threads";
+ private static final String STATUS_METRICS_PATH = "/status/metrics";
+
private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(WebServer.class);
private static final String OPTIONS_DESCRIBE_JS = "options.describe.js";
private static final String OPTIONS_DESCRIBE_TEMPLATE_JS =
"options.describe.template.js";
@@ -117,11 +131,12 @@ public class WebServer implements AutoCloseable {
public File getTmpJavaScriptDir() {
if (tmpJavaScriptDir == null) {
- tmpJavaScriptDir = Files.createTempDir();
+ tmpJavaScriptDir =
org.apache.drill.shaded.guava.com.google.common.io.Files.createTempDir();
tmpJavaScriptDir.deleteOnExit();
//Perform All auto generated files at this point
try {
generateOptionsDescriptionJSFile();
+ generateFunctionJS();
} catch (IOException e) {
logger.error("Unable to create temp dir for JavaScripts. {}", e);
}
@@ -177,7 +192,7 @@ public class WebServer implements AutoCloseable {
FilterHolder filterHolder = new FilterHolder(CrossOriginFilter.class);
filterHolder.setInitParameter("allowedOrigins", "*");
//Allowing CORS for metrics only
- webServerContext.addFilter(filterHolder, "/status/metrics", null);
+ webServerContext.addFilter(filterHolder, STATUS_METRICS_PATH, null);
embeddedJetty.setHandler(webServerContext);
ServerConnector connector = createConnector(port, acceptors, selectors);
@@ -215,8 +230,8 @@ public class WebServer implements AutoCloseable {
servletHolder.setInitOrder(1);
servletContextHandler.addServlet(servletHolder, "/*");
- servletContextHandler.addServlet(new ServletHolder(new
MetricsServlet(metrics)), "/status/metrics");
- servletContextHandler.addServlet(new ServletHolder(new
ThreadDumpServlet()), "/status/threads");
+ servletContextHandler.addServlet(new ServletHolder(new
MetricsServlet(metrics)), STATUS_METRICS_PATH);
+ servletContextHandler.addServlet(new ServletHolder(new
ThreadDumpServlet()), STATUS_THREADS_PATH);
final ServletHolder staticHolder = new ServletHolder("static",
DefaultServlet.class);
// Get resource URL for Drill static assets, based on where Drill icon is
located
@@ -456,14 +471,15 @@ public class WebServer implements AutoCloseable {
FileUtils.deleteDirectory(getTmpJavaScriptDir());
}
- private static final String FILE_CONTENT_FOOTER = "};";
-
- //Generate Options Description JavaScript
+ /**
+ * Generate Options Description JavaScript to serve http://drillhost/options
ACE library search features
+ * @throws IOException
+ */
private void generateOptionsDescriptionJSFile() throws IOException {
//Obtain list of Options & their descriptions
OptionManager optionManager =
this.drillbit.getContext().getOptionManager();
OptionList publicOptions = optionManager.getPublicOptionList();
- List<OptionValue> options = Lists.newArrayList(publicOptions);
+ List<OptionValue> options = new ArrayList<>(publicOptions);
Collections.sort(options);
int numLeftToWrite = options.size();
@@ -471,6 +487,7 @@ public class WebServer implements AutoCloseable {
InputStream optionsDescripTemplateStream =
Resource.newClassPathResource(OPTIONS_DESCRIBE_TEMPLATE_JS).getInputStream();
//Generated file
File optionsDescriptionFile = new File(getTmpJavaScriptDir(),
OPTIONS_DESCRIBE_JS);
+ final String file_content_footer = "};";
optionsDescriptionFile.deleteOnExit();
//Create a copy of a template and write with that!
java.nio.file.Files.copy(optionsDescripTemplateStream,
optionsDescriptionFile.toPath());
@@ -490,9 +507,56 @@ public class WebServer implements AutoCloseable {
writer.newLine();
}
}
- writer.append(FILE_CONTENT_FOOTER);
+ writer.append(file_content_footer);
writer.newLine();
writer.flush();
}
}
+
+ /**
+ * Generates ACE library javascript populated with list of available SQL
functions
+ * @throws IOException
+ */
+ private void generateFunctionJS() throws IOException {
+ //Naturally ordered set of function names
+ TreeSet<String> functionSet = new TreeSet<>();
+ //Extracting ONLY builtIn functions (i.e those already available)
+ List<FunctionHolder> builtInFuncHolderList =
this.drillbit.getContext().getFunctionImplementationRegistry().getLocalFunctionRegistry()
+ .getAllJarsWithFunctionsHolders().get(LocalFunctionRegistry.BUILT_IN);
+
+ //Build List of 'usable' functions (i.e. functions that start with an
alphabet and can be autocompleted by the ACE library)
+ //Example of 'unusable' functions would be operators like '<', '!'
+ int skipCount = 0;
+ for (FunctionHolder builtInFunctionHolder : builtInFuncHolderList) {
+ String name = builtInFunctionHolder.getName();
+ if (!name.contains(" ") && name.matches("([a-z]|[A-Z])\\w+") &&
!builtInFunctionHolder.getHolder().isInternal()) {
+ functionSet.add(name);
+ } else {
+ logger.debug("Non-alphabetic leading character. Function skipped : {}
", name);
+ skipCount++;
+ }
+ }
+ logger.debug("{} functions will not be available in WebUI", skipCount);
+
+ //Generated file
+ File functionsListFile = new File(getTmpJavaScriptDir(), ACE_MODE_SQL_JS);
+ functionsListFile.deleteOnExit();
+ //Template source Javascript file
+ try (InputStream aceModeSqlTemplateStream =
Resource.newClassPathResource(ACE_MODE_SQL_TEMPLATE_JS).getInputStream()) {
+ //Create a copy of a template and write with that!
+ java.nio.file.Files.copy(aceModeSqlTemplateStream,
functionsListFile.toPath());
+ }
+
+ //Construct String
+ String funcListString = String.join("|", functionSet);
+
+ Path path = Paths.get(functionsListFile.getPath());
+ try (Stream<String> lines = Files.lines(path)) {
+ List <String> replaced =
+ lines //Replacing first occurrence
+ .map(line -> line.replaceFirst(DRILL_FUNCTIONS_PLACEHOLDER,
funcListString))
+ .collect(Collectors.toList());
+ Files.write(path, replaced);
+ }
+ }
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/FunctionsIterator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/FunctionsIterator.java
index 86e8817..63798a2 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/FunctionsIterator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/FunctionsIterator.java
@@ -87,11 +87,12 @@ public class FunctionsIterator implements Iterator<Object> {
String registeredNames[] = dfh.getRegisteredNames();
String signature = dfh.getInputParameters();
String returnType = dfh.getReturnType().getMinorType().toString();
+ boolean isInternal = dfh.isInternal();
for (String name : registeredNames) {
//Generate a unique key for a function holder as
'functionName#functionSignature'
//Bumping capacity from default 16 to 64 (since key is expected to be
bigger, and reduce probability of resizing)
String funcSignatureKey = new
StringBuilder(64).append(name).append('#').append(signature).toString();
- functionMap.put(funcSignatureKey, new FunctionInfo(name, signature,
returnType, jarName));
+ functionMap.put(funcSignatureKey, new FunctionInfo(name, signature,
returnType, jarName, isInternal));
}
}
@@ -117,12 +118,19 @@ public class FunctionsIterator implements
Iterator<Object> {
public final String returnType;
@NonNullable
public final String source;
+ @NonNullable
+ public final boolean internal;
public FunctionInfo(String funcName, String funcSignature, String
funcReturnType, String jarName) {
+ this(funcName, funcSignature, funcReturnType, jarName, false);
+ }
+
+ public FunctionInfo(String funcName, String funcSignature, String
funcReturnType, String jarName, boolean isInternal) {
this.name = funcName;
this.signature = funcSignature;
this.returnType = funcReturnType;
this.source = jarName;
+ this.internal = isInternal;
}
}
}
diff --git
a/exec/java-exec/src/main/resources/rest/static/js/ace-code-editor/mode-sql.js
b/exec/java-exec/src/main/resources/ace.mode-sql.template.js
similarity index 80%
rename from
exec/java-exec/src/main/resources/rest/static/js/ace-code-editor/mode-sql.js
rename to exec/java-exec/src/main/resources/ace.mode-sql.template.js
index ceb47c8..057d8df 100644
---
a/exec/java-exec/src/main/resources/rest/static/js/ace-code-editor/mode-sql.js
+++ b/exec/java-exec/src/main/resources/ace.mode-sql.template.js
@@ -31,22 +31,9 @@ var SqlHighlightRules = function() {
"true|false"
);
- //Drill-specific
+ //Drill-specific (auto-generated: See DRILL-6084)
var builtinFunctions = (
- //Math and Trignometric
-
"abs|cbrt|ceil|ceiling|degrees|e|exp|floor|log|log|log10|lshift|mod|negative|pi|pow|radians|rand|round|round|rshift|sign|sqrt|trunc|trunc|"
+
- "sin|cos|tan|asin|acos|atan|sinh|cosh|tanh|" +
- //datatype conversion
- "cast|convert_to|convert_from|string_binary|binary_string|" +
- //time-conversion
- "to_char|to_date|to_number|to_timestamp|to_timestamp|" +
-
"age|extract|current_date|current_time|current_timestamp|date_add|date_part|date_sub|localtime|localtimestamp|now|timeofday|unix_timestamp|"
+
- //string manipulation
-
"byte_substr|char_length|concat|ilike|initcap|length|lower|lpad|ltrim|position|regexp_replace|rpad|rtrim|strpos|substr|trim|upper|"
+
- //statistical
-
"avg|count|max|min|sum|stddev|stddev_pop|stddev_samp|variance|var_pop|var_samp|"
+
- //null-handling
- "coalesce|nullif"
+ "__DRILL_FUNCTIONS__"
);
//Drill-specific
diff --git a/exec/java-exec/src/main/resources/rest/query/query.ftl
b/exec/java-exec/src/main/resources/rest/query/query.ftl
index 936b1a5..078333e 100644
--- a/exec/java-exec/src/main/resources/rest/query/query.ftl
+++ b/exec/java-exec/src/main/resources/rest/query/query.ftl
@@ -25,7 +25,8 @@
</#if>
<!-- Ace Libraries for Syntax Formatting -->
<script src="/static/js/ace-code-editor/ace.js" type="text/javascript"
charset="utf-8"></script>
- <script src="/static/js/ace-code-editor/mode-sql.js" type="text/javascript"
charset="utf-8"></script>
+ <!-- Disabled in favour of dynamic: script
src="/static/js/ace-code-editor/mode-sql.js" type="text/javascript"
charset="utf-8" -->
+ <script src="/dynamic/mode-sql.js" type="text/javascript"
charset="utf-8"></script>
<script src="/static/js/ace-code-editor/ext-language_tools.js"
type="text/javascript" charset="utf-8"></script>
<script src="/static/js/ace-code-editor/theme-sqlserver.js"
type="text/javascript" charset="utf-8"></script>
<script src="/static/js/ace-code-editor/snippets/sql.js"
type="text/javascript" charset="utf-8"></script>
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java
index 5c9990a..2ed5f6e 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java
@@ -82,6 +82,13 @@ public class TestSystemTable extends PlanTestBase {
}
@Test
+ public void testInternalFunctionsTable() throws Exception {
+ String query = "select internal, count(*) from sys.functions group by
internal";
+ //Testing a mix of public and internal functions defined in
FunctionTemplate
+ assertEquals(2, testSql(query));
+ }
+
+ @Test
public void profilesTable() throws Exception {
test("select * from sys.profiles");
}
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
index ce58cb0..2e785b7 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
@@ -240,7 +240,7 @@ public class TestMetadataProvider extends BaseTestQuery {
assertEquals(RequestStatus.OK, resp.getStatus());
List<ColumnMetadata> columns = resp.getColumnsList();
- assertEquals(139, columns.size());
+ assertEquals(140, columns.size());
// too many records to verify the output.
}
