This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 2fbcab0b8bc6e8e47602613212e1a19dcb01c671 Author: Ian Maxon <[email protected]> AuthorDate: Tue Mar 16 19:32:42 2021 -0700 [NO ISSUE] UDF API improvements - user model changes: yes - storage format changes: yes - interface changes: yes Details: - Make a GET on / return all installed libraries on that node - Make GET return JSON wrapped errors - Roll all modification functionality into POST - Make dataverse, library name, and library language specified as multipart form data fields (dataverse, name, and type) - Make library data a named field in the multipart post data body (data) - Allow multipart dataverse names - Add support for multipart/form-data parameters to TestExecutor - Fix resource leaks - Require either python.cmd to be set or python.cmd.autolocate to be true to automatically attempt to locate python interpreter Change-Id: Ib6e6ce7debc9c2e07d24163542c1f98886792165 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9565 Reviewed-by: Ian Maxon <[email protected]> Reviewed-by: Dmitry Lychagin <[email protected]> Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> --- .../api/http/server/AbstractNCUdfServlet.java | 206 ++++++++++++++++++--- .../asterix/api/http/server/BasicAuthServlet.java | 13 +- .../asterix/api/http/server/NCUdfApiServlet.java | 193 ++++++++++--------- .../api/http/server/NCUdfRecoveryServlet.java | 5 +- .../asterix/hyracks/bootstrap/NCApplication.java | 6 +- .../asterix/app/external/ExternalUDFLibrarian.java | 46 +++-- .../app/external/IExternalUDFLibrarian.java | 7 +- .../apache/asterix/test/common/TestExecutor.java | 145 +++++++++++++-- asterixdb/asterix-app/src/test/resources/cc.conf | 1 + .../bad-ext-function-ddl-1.2.lib.sqlpp | 2 +- .../create-or-replace-function-1.2.lib.sqlpp | 2 +- .../deterministic/deterministic.1.lib.sqlpp | 2 +- .../exception_create_system_library.1.lib.sqlpp | 2 +- .../getCapital/getCapital.1.lib.sqlpp | 2 +- .../getCapital_open/getCapital_open.1.lib.sqlpp | 2 +- .../library_list_api_multipart.1.post.http} | 8 +- .../library_list_api_multipart.2.post.http} | 9 +- .../library_list_api_multipart.3.post.http} | 7 +- .../library_list_api_multipart.4.post.http} | 8 +- .../library_list_api_multipart.5.post.http} | 7 +- .../keyword_detector/keyword_detector.1.lib.sqlpp | 2 +- .../library_list_api.0.ddl.sqlpp} | 4 +- .../library_list_api.1.post.http} | 8 +- .../library_list_api.2.get.http} | 4 +- .../library_list_api_multipart.0.ddl.sqlpp} | 9 +- .../library_list_api_multipart.1.post.http} | 9 +- .../library_list_api_multipart.2.post.http} | 8 +- .../library_list_api_multipart.3.post.http} | 10 +- .../library_list_api_multipart.4.post.http} | 10 +- .../library_list_api_multipart.5.get.http} | 4 +- .../my_array_sum/my_array_sum.1.lib.sqlpp | 2 +- .../mysentiment/mysentiment.1.lib.sqlpp | 2 +- .../mysentiment_multipart.0.ddl.sqlpp} | 4 +- .../mysentiment_multipart.1.lib.sqlpp} | 3 +- .../mysentiment_multipart.2.ddl.sqlpp} | 5 +- .../mysentiment_multipart.3.query.sqlpp} | 4 +- .../mysentiment_multipart.4.query.sqlpp} | 4 +- .../mysentiment_multipart.5.query.sqlpp} | 4 +- .../mysentiment_multipart.6.ddl.sqlpp} | 3 +- .../external-library/mysum/mysum.1.lib.sqlpp | 2 +- .../mysum_bad_credential.1.lib.sqlpp | 2 +- .../mysum_bad_credential.2.lib.sqlpp | 2 +- .../mysum_bad_credential.3.lib.sqlpp | 2 +- .../mysum_dropinuse/mysum_dropinuse.1.lib.sqlpp | 2 +- .../py_function_error.1.lib.sqlpp | 2 +- .../py_nested_access/py_nested_access.1.lib.sqlpp | 2 +- .../python-fn-escape/python-fn-escape.1.lib.sqlpp | 2 +- .../return_invalid_type.1.lib.sqlpp | 2 +- .../type_validation/type_validation.1.lib.sqlpp | 2 +- .../udf_metadata/udf_metadata.1.lib.sqlpp | 4 +- .../upperCase/upperCase.1.lib.sqlpp | 2 +- .../exception_create_system_adapter.1.lib.sqlpp | 2 +- ...feed-with-external-adapter-cross-dv.1.lib.sqlpp | 2 +- .../feed-with-external-adapter.1.lib.sqlpp | 2 +- .../feed-with-external-function.1.lib.sqlpp | 2 +- .../invalid-library-params.1.regexjson | 3 + .../library_list_api/library_list_ap1.1.regexjson | 1 + .../library_list_api/library_list_ap1.2.regexjson | 5 + .../library_list_ap1.1.regexjson | 1 + .../library_list_ap1.2.regexjson | 1 + .../library_list_ap1.3.regexjson | 1 + .../library_list_ap1.4.regexjson | 1 + .../library_list_ap1.5.regexjson | 20 ++ .../resources/runtimets/testsuite_it_python.xml | 5 + .../resources/runtimets/testsuite_it_sqlpp.xml | 20 ++ .../asterix/common/library/ILibraryManager.java | 6 + .../asterix/common/library/LibraryDescriptor.java | 8 +- .../src/main/resources/asx_errormsg/en.properties | 2 +- .../external/library/ExternalLibraryManager.java | 76 +++++++- .../ExternalScalarPythonFunctionEvaluator.java | 13 +- .../LibraryDeployPrepareOperatorDescriptor.java | 8 - .../external/util/ExternalLibraryUtils.java | 18 +- .../src/main/resources/Catalog.xsd | 2 + .../control/common/controllers/NCConfig.java | 5 +- 74 files changed, 785 insertions(+), 217 deletions(-) diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractNCUdfServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractNCUdfServlet.java index 54e972e..a234b9f 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractNCUdfServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractNCUdfServlet.java @@ -22,40 +22,46 @@ import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_CONNEC import static org.apache.asterix.common.functions.ExternalFunctionLanguage.JAVA; import static org.apache.asterix.common.functions.ExternalFunctionLanguage.PYTHON; +import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.nio.file.Files; import java.nio.file.OpenOption; import java.nio.file.Path; -import java.util.Arrays; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.ConcurrentMap; import org.apache.asterix.common.api.IApplicationContext; import org.apache.asterix.common.api.INcApplicationContext; +import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.common.exceptions.RuntimeDataException; import org.apache.asterix.common.functions.ExternalFunctionLanguage; -import org.apache.asterix.common.library.LibraryDescriptor; import org.apache.asterix.common.metadata.DataverseName; +import org.apache.asterix.compiler.provider.ILangCompilationProvider; +import org.apache.asterix.lang.common.base.IParserFactory; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; -import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.api.application.INCServiceContext; import org.apache.hyracks.api.client.IHyracksClientConnection; import org.apache.hyracks.api.exceptions.IFormattedException; import org.apache.hyracks.control.common.work.SynchronizableWork; import org.apache.hyracks.control.nc.NodeControllerService; -import org.apache.hyracks.http.api.IServletRequest; import org.apache.hyracks.http.api.IServletResponse; import org.apache.hyracks.http.server.AbstractServlet; import org.apache.hyracks.http.server.utils.HttpUtil; import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpScheme; +import io.netty.handler.codec.http.multipart.FileUpload; +import io.netty.handler.codec.http.multipart.HttpPostRequestDecoder; +import io.netty.handler.codec.http.multipart.InterfaceHttpData; +import io.netty.handler.codec.http.multipart.MixedAttribute; public abstract class AbstractNCUdfServlet extends AbstractServlet { + private final IParserFactory parserFactory; INcApplicationContext appCtx; INCServiceContext srvCtx; @@ -63,13 +69,81 @@ public abstract class AbstractNCUdfServlet extends AbstractServlet { private final HttpScheme httpServerProtocol; private final int httpServerPort; + public static final String GET_UDF_DIST_ENDPOINT = "/dist"; + public static final String DATAVERSE_PARAMETER = "dataverse"; + public static final String NAME_PARAMETER = "name"; + public static final String TYPE_PARAMETER = "type"; + public static final String DELETE_PARAMETER = "delete"; + public static final String IFEXISTS_PARAMETER = "ifexists"; + public static final String DATA_PARAMETER = "data"; + + protected enum LibraryOperation { + UPSERT, + DELETE + } + + protected final static class LibraryUploadData { + + final LibraryOperation op; + final DataverseName dataverse; + final String name; + final ExternalFunctionLanguage type; + final boolean replaceIfExists; + final FileUpload fileUpload; + + private LibraryUploadData(LibraryOperation op, List<InterfaceHttpData> dataverse, MixedAttribute name, + MixedAttribute type, boolean replaceIfExists, InterfaceHttpData fileUpload) throws IOException { + this.op = op; + List<String> dataverseParts = new ArrayList<>(dataverse.size()); + for (InterfaceHttpData attr : dataverse) { + dataverseParts.add(((MixedAttribute) attr).getValue()); + } + this.dataverse = DataverseName.create(dataverseParts); + this.name = name.getValue(); + this.type = type != null ? getLanguageByTypeParameter(type.getValue()) : null; + this.replaceIfExists = replaceIfExists; + this.fileUpload = (FileUpload) fileUpload; + } + + private LibraryUploadData(LibraryOperation op, DataverseName dataverse, MixedAttribute name, + MixedAttribute type, boolean replaceIfExists, InterfaceHttpData fileUpload) throws IOException { + this.op = op; + this.dataverse = dataverse; + this.name = name.getValue(); + this.type = type != null ? getLanguageByTypeParameter(type.getValue()) : null; + this.replaceIfExists = replaceIfExists; + this.fileUpload = (FileUpload) fileUpload; + } + + public static LibraryUploadData libraryCreationUploadData(List<InterfaceHttpData> dataverse, + MixedAttribute name, MixedAttribute type, InterfaceHttpData fileUpload) throws IOException { + return new LibraryUploadData(LibraryOperation.UPSERT, dataverse, name, type, true, fileUpload); + } + + public static LibraryUploadData libraryDeletionUploadData(List<InterfaceHttpData> dataverse, + MixedAttribute name, boolean replaceIfExists) throws IOException { + return new LibraryUploadData(LibraryOperation.DELETE, dataverse, name, null, replaceIfExists, null); + } + + public static LibraryUploadData libraryCreationUploadData(DataverseName dataverse, MixedAttribute name, + MixedAttribute type, InterfaceHttpData fileUpload) throws IOException { + return new LibraryUploadData(LibraryOperation.UPSERT, dataverse, name, type, true, fileUpload); + } + + public static LibraryUploadData libraryDeletionUploadData(DataverseName dataverse, MixedAttribute name, + boolean replaceIfExists) throws IOException { + return new LibraryUploadData(LibraryOperation.DELETE, dataverse, name, null, replaceIfExists, null); + } + } + public AbstractNCUdfServlet(ConcurrentMap<String, Object> ctx, String[] paths, IApplicationContext appCtx, - HttpScheme httpServerProtocol, int httpServerPort) { + ILangCompilationProvider compilationProvider, HttpScheme httpServerProtocol, int httpServerPort) { super(ctx, paths); this.plainAppCtx = appCtx; this.httpServerProtocol = httpServerProtocol; this.httpServerPort = httpServerPort; + this.parserFactory = compilationProvider.getParserFactory(); } void readFromFile(Path filePath, IServletResponse response, String contentType, OpenOption opt) throws Exception { @@ -103,7 +177,7 @@ public abstract class AbstractNCUdfServlet extends AbstractServlet { } URI createDownloadURI(Path file) throws Exception { - String path = paths[0].substring(0, trims[0]) + '/' + file.getFileName(); + String path = paths[0].substring(0, trims[0]) + GET_UDF_DIST_ENDPOINT + '/' + file.getFileName(); String host = getHyracksClientConnection().getHost(); return new URI(httpServerProtocol.toString(), null, host, httpServerPort, path, null, null); } @@ -116,25 +190,105 @@ public abstract class AbstractNCUdfServlet extends AbstractServlet { return hcc; } - Pair<DataverseName, String> parseLibraryName(IServletRequest request) throws IllegalArgumentException { - String[] path = StringUtils.split(localPath(request), '/'); - int ln = path.length; - if (ln < 2) { - return null; + protected String getDisplayFormDataverseParameter() { + return null; + } + + protected String getDataverseParameter() { + return DATAVERSE_PARAMETER; + } + + private boolean isNotAttribute(InterfaceHttpData field) { + return field == null || !field.getHttpDataType().equals(InterfaceHttpData.HttpDataType.Attribute); + } + + private boolean areNotAttributes(List<InterfaceHttpData> fields) { + return fields == null || fields.stream().map(InterfaceHttpData::getHttpDataType) + .anyMatch(httpDataType -> !httpDataType.equals(InterfaceHttpData.HttpDataType.Attribute)); + } + + protected LibraryUploadData decodeMultiPartLibraryOptions(HttpPostRequestDecoder requestDecoder) + throws IOException, CompilationException { + List<InterfaceHttpData> dataverseAttributeParts = requestDecoder.getBodyHttpDatas(DATAVERSE_PARAMETER); + InterfaceHttpData displayFormDataverseAttribute = null; + if (getDisplayFormDataverseParameter() != null) { + displayFormDataverseAttribute = requestDecoder.getBodyHttpData(getDisplayFormDataverseParameter()); + } + if (displayFormDataverseAttribute != null && dataverseAttributeParts != null) { + throw RuntimeDataException.create(ErrorCode.PARAMETERS_NOT_ALLOWED_AT_SAME_TIME, + getDisplayFormDataverseParameter(), getDataverseParameter()); + } + InterfaceHttpData nameAtrribute = requestDecoder.getBodyHttpData(NAME_PARAMETER); + InterfaceHttpData typeAttribute = requestDecoder.getBodyHttpData(TYPE_PARAMETER); + InterfaceHttpData deleteAttribute = requestDecoder.getBodyHttpData(DELETE_PARAMETER); + InterfaceHttpData replaceIfExistsAttribute = requestDecoder.getBodyHttpData(IFEXISTS_PARAMETER); + if ((isNotAttribute(displayFormDataverseAttribute)) && (areNotAttributes(dataverseAttributeParts))) { + throw RuntimeDataException.create(ErrorCode.PARAMETERS_REQUIRED, getDataverseParameter()); + } else if (isNotAttribute(nameAtrribute)) { + throw RuntimeDataException.create(ErrorCode.PARAMETERS_REQUIRED, NAME_PARAMETER); + } else if ((typeAttribute == null && deleteAttribute == null)) { + throw RuntimeDataException.create(ErrorCode.PARAMETERS_REQUIRED, + TYPE_PARAMETER + " or " + DELETE_PARAMETER); + } else if (typeAttribute != null && deleteAttribute != null) { + throw RuntimeDataException.create(ErrorCode.PARAMETERS_NOT_ALLOWED_AT_SAME_TIME, TYPE_PARAMETER, + DELETE_PARAMETER); + } + + if (!isNotAttribute(deleteAttribute)) { + boolean replace = false; + if (replaceIfExistsAttribute != null) { + replace = Boolean.TRUE.toString() + .equalsIgnoreCase(((MixedAttribute) replaceIfExistsAttribute).getValue()); + } + if (displayFormDataverseAttribute == null) { + return LibraryUploadData.libraryDeletionUploadData(dataverseAttributeParts, + (MixedAttribute) nameAtrribute, replace); + } else { + DataverseName dataverseName = DataverseName + .create(parserFactory.createParser(((MixedAttribute) displayFormDataverseAttribute).getValue()) + .parseMultipartIdentifier()); + return LibraryUploadData.libraryDeletionUploadData(dataverseName, (MixedAttribute) nameAtrribute, + replace); + } + } else if (!isNotAttribute(typeAttribute)) { + InterfaceHttpData libraryData = requestDecoder.getBodyHttpData(DATA_PARAMETER); + if (libraryData == null) { + throw RuntimeDataException.create(ErrorCode.PARAMETERS_REQUIRED, DATA_PARAMETER); + } else if (!libraryData.getHttpDataType().equals(InterfaceHttpData.HttpDataType.FileUpload)) { + throw RuntimeDataException.create(ErrorCode.INVALID_REQ_PARAM_VAL, DATA_PARAMETER, + libraryData.getHttpDataType()); + } + LibraryUploadData uploadData; + if (displayFormDataverseAttribute == null) { + uploadData = LibraryUploadData.libraryCreationUploadData(dataverseAttributeParts, + (MixedAttribute) nameAtrribute, (MixedAttribute) typeAttribute, libraryData); + } else { + DataverseName dataverseName = DataverseName + .create(parserFactory.createParser(((MixedAttribute) displayFormDataverseAttribute).getValue()) + .parseMultipartIdentifier()); + uploadData = LibraryUploadData.libraryCreationUploadData(dataverseName, (MixedAttribute) nameAtrribute, + (MixedAttribute) typeAttribute, libraryData); + } + if (uploadData.type == null) { + throw RuntimeDataException.create(ErrorCode.LIBRARY_EXTERNAL_FUNCTION_UNSUPPORTED_KIND, + ((MixedAttribute) typeAttribute).getValue()); + } + return uploadData; + } else { + if (!typeAttribute.getHttpDataType().equals(InterfaceHttpData.HttpDataType.Attribute)) { + throw RuntimeDataException.create(ErrorCode.PARAMETERS_REQUIRED, TYPE_PARAMETER); + } + throw RuntimeDataException.create(ErrorCode.PARAMETERS_REQUIRED, DELETE_PARAMETER); } - String libraryName = path[ln - 1]; - DataverseName dataverseName = DataverseName.create(Arrays.asList(path), 0, ln - 1); - return new Pair<>(dataverseName, libraryName); } - static ExternalFunctionLanguage getLanguageByFileExtension(String fileExtension) { - switch (fileExtension) { - case LibraryDescriptor.FILE_EXT_ZIP: - return JAVA; - case LibraryDescriptor.FILE_EXT_PYZ: - return PYTHON; - default: - return null; + static ExternalFunctionLanguage getLanguageByTypeParameter(String lang) { + if (lang.equalsIgnoreCase(JAVA.name())) { + return JAVA; + } else if (lang.equalsIgnoreCase(PYTHON.name())) { + return PYTHON; + } else { + return null; } } @@ -142,9 +296,15 @@ public abstract class AbstractNCUdfServlet extends AbstractServlet { if (IFormattedException.matchesAny(e, ErrorCode.UNKNOWN_DATAVERSE, ErrorCode.UNKNOWN_LIBRARY)) { return HttpResponseStatus.NOT_FOUND; } + if (IFormattedException.matchesAny(e, ErrorCode.LIBRARY_EXTERNAL_FUNCTION_UNKNOWN_KIND, + ErrorCode.LIBRARY_EXTERNAL_FUNCTION_UNSUPPORTED_KIND, ErrorCode.INVALID_REQ_PARAM_VAL, + ErrorCode.PARAMETERS_REQUIRED)) { + return HttpResponseStatus.BAD_REQUEST; + } if (e instanceof AlgebricksException) { return HttpResponseStatus.BAD_REQUEST; } return HttpResponseStatus.INTERNAL_SERVER_ERROR; } + } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/BasicAuthServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/BasicAuthServlet.java index e062cdc..9dc971a 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/BasicAuthServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/BasicAuthServlet.java @@ -130,7 +130,7 @@ public class BasicAuthServlet implements IServlet { return false; } String providedUsername = providedCredentials[0]; - String storedPw = getStoredCredentials(request).get(providedUsername); + String storedPw = getCredential(providedUsername, request); if (storedPw == null) { LOGGER.debug("Invalid username"); return false; @@ -144,8 +144,15 @@ public class BasicAuthServlet implements IServlet { } } - protected Map<String, String> getStoredCredentials(IServletRequest request) { - return request.getHttpRequest().method().equals(HttpMethod.GET) ? ephemeralCredentials : storedCredentials; + private String getCredential(String username, IServletRequest request) { + String credential = storedCredentials.get(username); + if (credential != null) { + return credential; + } else if (request != null && request.getHttpRequest().method().equals(HttpMethod.GET)) { + return ephemeralCredentials.get(username); + } else { + return null; + } } public static String hashPassword(String password) { diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java index 0645870..f164938 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java @@ -19,6 +19,7 @@ package org.apache.asterix.api.http.server; import static org.apache.asterix.api.http.server.ServletConstants.SYS_AUTH_HEADER; +import static org.apache.asterix.common.library.LibraryDescriptor.FIELD_HASH; import java.io.FileOutputStream; import java.io.IOException; @@ -31,7 +32,10 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.security.DigestOutputStream; import java.security.MessageDigest; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; @@ -44,6 +48,7 @@ import org.apache.asterix.common.api.INcApplicationContext; import org.apache.asterix.common.api.IReceptionist; import org.apache.asterix.common.api.IRequestReference; import org.apache.asterix.common.functions.ExternalFunctionLanguage; +import org.apache.asterix.common.library.ILibraryManager; import org.apache.asterix.common.messaging.api.ICcAddressedMessage; import org.apache.asterix.common.messaging.api.INCMessageBroker; import org.apache.asterix.common.messaging.api.MessageFuture; @@ -53,21 +58,22 @@ import org.apache.asterix.external.util.ExternalLibraryUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; -import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.http.api.IServletRequest; import org.apache.hyracks.http.api.IServletResponse; import org.apache.hyracks.http.server.utils.HttpUtil; +import org.apache.hyracks.util.JSONUtil; import org.apache.hyracks.util.file.FileUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; + import io.netty.buffer.ByteBufInputStream; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpScheme; -import io.netty.handler.codec.http.multipart.FileUpload; import io.netty.handler.codec.http.multipart.HttpPostRequestDecoder; -import io.netty.handler.codec.http.multipart.InterfaceHttpData; public class NCUdfApiServlet extends AbstractNCUdfServlet { @@ -76,12 +82,13 @@ public class NCUdfApiServlet extends AbstractNCUdfServlet { protected Path workingDir; protected String sysAuthHeader; + private ILibraryManager libraryManager; private static final Logger LOGGER = LogManager.getLogger(); public NCUdfApiServlet(ConcurrentMap<String, Object> ctx, String[] paths, IApplicationContext appCtx, ILangCompilationProvider compilationProvider, HttpScheme httpServerProtocol, int httpServerPort) { - super(ctx, paths, appCtx, httpServerProtocol, httpServerPort); + super(ctx, paths, appCtx, compilationProvider, httpServerProtocol, httpServerPort); this.compilationProvider = compilationProvider; this.receptionist = appCtx.getReceptionist(); } @@ -89,6 +96,7 @@ public class NCUdfApiServlet extends AbstractNCUdfServlet { @Override public void init() throws IOException { appCtx = (INcApplicationContext) plainAppCtx; + this.libraryManager = appCtx.getLibraryManager(); srvCtx = this.appCtx.getServiceContext(); workingDir = Paths.get(appCtx.getLibraryManager().getDistributionDir().getAbsolutePath()).normalize(); initAuth(); @@ -119,27 +127,26 @@ public class NCUdfApiServlet extends AbstractNCUdfServlet { private void doCreate(DataverseName dataverseName, String libraryName, ExternalFunctionLanguage language, String hash, URI downloadURI, boolean replaceIfExists, String sysAuthHeader, - IRequestReference requestReference, IServletRequest request, IServletResponse response) throws Exception { + IRequestReference requestReference, IServletRequest request) throws Exception { INCMessageBroker ncMb = (INCMessageBroker) srvCtx.getMessageBroker(); MessageFuture responseFuture = ncMb.registerMessageFuture(); CreateLibraryRequestMessage req = new CreateLibraryRequestMessage(srvCtx.getNodeId(), responseFuture.getFutureId(), dataverseName, libraryName, language, hash, downloadURI, replaceIfExists, sysAuthHeader, requestReference, additionalHttpHeadersFromRequest(request)); - sendMessage(req, responseFuture, requestReference, request, response); + sendMessage(req, responseFuture); } private void doDrop(DataverseName dataverseName, String libraryName, boolean replaceIfExists, - IRequestReference requestReference, IServletRequest request, IServletResponse response) throws Exception { + IRequestReference requestReference, IServletRequest request) throws Exception { INCMessageBroker ncMb = (INCMessageBroker) srvCtx.getMessageBroker(); MessageFuture responseFuture = ncMb.registerMessageFuture(); DropLibraryRequestMessage req = new DropLibraryRequestMessage(srvCtx.getNodeId(), responseFuture.getFutureId(), dataverseName, libraryName, replaceIfExists, requestReference, additionalHttpHeadersFromRequest(request)); - sendMessage(req, responseFuture, requestReference, request, response); + sendMessage(req, responseFuture); } - private void sendMessage(ICcAddressedMessage requestMessage, MessageFuture responseFuture, - IRequestReference requestReference, IServletRequest request, IServletResponse response) throws Exception { + private void sendMessage(ICcAddressedMessage requestMessage, MessageFuture responseFuture) throws Exception { // Running on NC -> send 'execute' message to CC INCMessageBroker ncMb = (INCMessageBroker) srvCtx.getMessageBroker(); InternalRequestResponse responseMsg; @@ -165,107 +172,125 @@ public class NCUdfApiServlet extends AbstractNCUdfServlet { @Override protected void get(IServletRequest request, IServletResponse response) throws Exception { String localPath = localPath(request); - while (localPath.startsWith("/")) { - localPath = localPath.substring(1); - } - if (localPath.isEmpty()) { - response.setStatus(HttpResponseStatus.BAD_REQUEST); - return; - } - Path filePath = workingDir.resolve(localPath).normalize(); - if (!filePath.startsWith(workingDir)) { - response.setStatus(HttpResponseStatus.BAD_REQUEST); - return; + try { + if (localPath.equals("/") || localPath.equals("")) { + //TODO: nicer way to get this into display form? + Map<DataverseName, Map<String, String>> dvToLibHashes = + ExternalLibraryUtils.produceLibraryListing(libraryManager); + List<Map<String, Object>> libraryList = new ArrayList<>(); + for (Map.Entry<DataverseName, Map<String, String>> dvAndLibs : dvToLibHashes.entrySet()) { + for (Map.Entry<String, String> libsInDv : dvAndLibs.getValue().entrySet()) { + Map<String, Object> libraryEntry = new HashMap<>(); + List<String> dvParts = dvAndLibs.getKey().getParts(); + String dvKey = getDisplayFormDataverseParameter() == null ? getDataverseParameter() + : getDisplayFormDataverseParameter(); + libraryEntry.put(dvKey, dvParts.size() > 1 ? dvParts : dvAndLibs.getKey().toString()); + libraryEntry.put(NAME_PARAMETER, libsInDv.getKey()); + libraryEntry.put(FIELD_HASH, libsInDv.getValue()); + libraryList.add(libraryEntry); + } + } + JsonNode libraryListing = OBJECT_MAPPER.valueToTree(libraryList); + response.setStatus(HttpResponseStatus.OK); + HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request); + PrintWriter responseWriter = response.writer(); + JSONUtil.writeNode(responseWriter, libraryListing); + responseWriter.flush(); + } else if (localPath(request).startsWith(GET_UDF_DIST_ENDPOINT)) { + localPath = localPath(request).substring(GET_UDF_DIST_ENDPOINT.length()); + while (localPath.startsWith("/")) { + localPath = localPath.substring(1); + } + if (localPath.isEmpty()) { + response.setStatus(HttpResponseStatus.BAD_REQUEST); + return; + } + Path filePath = workingDir.resolve(localPath).normalize(); + if (!filePath.startsWith(workingDir)) { + response.setStatus(HttpResponseStatus.BAD_REQUEST); + return; + } + readFromFile(filePath, response, HttpUtil.ContentType.APPLICATION_OCTET_STREAM, null); + } else { + response.setStatus(HttpResponseStatus.NOT_FOUND); + } + } catch (Exception e) { + response.setStatus(toHttpErrorStatus(e)); + PrintWriter responseWriter = response.writer(); + Map<String, String> error = Collections.singletonMap("error", e.getMessage()); + String errorJson = ""; + try { + errorJson = OBJECT_MAPPER.writeValueAsString(error); + } catch (JsonProcessingException ex) { + responseWriter.write("{ \"error\": \"Unable to process error message!\" }"); + } + responseWriter.write(errorJson); + responseWriter.flush(); + LOGGER.error("Error reading library", e); } - readFromFile(filePath, response, HttpUtil.ContentType.APPLICATION_OCTET_STREAM, null); } @Override protected void post(IServletRequest request, IServletResponse response) { HttpRequest httpRequest = request.getHttpRequest(); - Pair<DataverseName, String> libraryName = parseLibraryName(request); - if (libraryName == null) { - response.setStatus(HttpResponseStatus.BAD_REQUEST); - return; - } Path libraryTempFile = null; FileOutputStream libTmpOut = null; HttpPostRequestDecoder requestDecoder = new HttpPostRequestDecoder(httpRequest); try { - if (!requestDecoder.hasNext() || requestDecoder.getBodyHttpDatas().size() != 1) { - response.setStatus(HttpResponseStatus.BAD_REQUEST); - return; - } - InterfaceHttpData httpData = requestDecoder.getBodyHttpDatas().get(0); - if (!httpData.getHttpDataType().equals(InterfaceHttpData.HttpDataType.FileUpload)) { - response.setStatus(HttpResponseStatus.BAD_REQUEST); - return; - } - FileUpload fileUpload = (FileUpload) httpData; - String fileExt = FilenameUtils.getExtension(fileUpload.getFilename()); - ExternalFunctionLanguage language = getLanguageByFileExtension(fileExt); - if (language == null) { - response.setStatus(HttpResponseStatus.BAD_REQUEST); - return; - } - try { - IRequestReference requestReference = receptionist.welcome(request); + LibraryUploadData uploadData = decodeMultiPartLibraryOptions(requestDecoder); + IRequestReference requestReference = receptionist.welcome(request); + if (uploadData.op == LibraryOperation.UPSERT) { + ExternalFunctionLanguage language = uploadData.type; + String fileExt = FilenameUtils.getExtension(uploadData.fileUpload.getFilename()); libraryTempFile = Files.createTempFile(workingDir, "lib_", '.' + fileExt); if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Created temporary file " + libraryTempFile + " for library " + libraryName.first + "." - + libraryName.second); + LOGGER.debug("Created temporary file " + libraryTempFile + " for library " + uploadData.dataverse + + "." + uploadData.name); } MessageDigest digest = MessageDigest.getInstance("MD5"); libTmpOut = new FileOutputStream(libraryTempFile.toFile()); - OutputStream outStream = new DigestOutputStream(libTmpOut, digest); - InputStream uploadInput = new ByteBufInputStream(((FileUpload) httpData).getByteBuf()); - IOUtils.copyLarge(uploadInput, outStream); - outStream.close(); + try (OutputStream os = new DigestOutputStream(libTmpOut, digest); + InputStream ui = new ByteBufInputStream((uploadData.fileUpload).getByteBuf())) { + IOUtils.copyLarge(ui, os); + } URI downloadURI = createDownloadURI(libraryTempFile); - doCreate(libraryName.first, libraryName.second, language, + doCreate(uploadData.dataverse, uploadData.name, language, ExternalLibraryUtils.digestToHexString(digest), downloadURI, true, sysAuthHeader, - requestReference, request, response); - response.setStatus(HttpResponseStatus.OK); - } catch (Exception e) { - response.setStatus(toHttpErrorStatus(e)); - PrintWriter responseWriter = response.writer(); - responseWriter.write(e.getMessage()); - responseWriter.flush(); - LOGGER.error("Error creating/updating library " + libraryName.first + "." + libraryName.second, e); + requestReference, request); + } else if (uploadData.op == LibraryOperation.DELETE) { + doDrop(uploadData.dataverse, uploadData.name, uploadData.replaceIfExists, requestReference, request); } + response.setStatus(HttpResponseStatus.OK); + PrintWriter responseWriter = response.writer(); + String emptyJson = "{}"; + responseWriter.write(emptyJson); + responseWriter.flush(); + } catch (Exception e) { + response.setStatus(toHttpErrorStatus(e)); + PrintWriter responseWriter = response.writer(); + Map<String, String> error = Collections.singletonMap("error", e.getMessage()); + String errorJson = ""; + try { + errorJson = OBJECT_MAPPER.writeValueAsString(error); + } catch (JsonProcessingException ex) { + responseWriter.write("{ \"error\": \"Unable to process error message!\" }"); + } + responseWriter.write(errorJson); + responseWriter.flush(); + LOGGER.error("Error modifying library", e); } finally { requestDecoder.destroy(); - if (libraryTempFile != null) { - try { + try { + if (libraryTempFile != null) { if (libTmpOut != null) { libTmpOut.close(); } Files.deleteIfExists(libraryTempFile); - } catch (IOException e) { - LOGGER.warn("Could not delete temporary file " + libraryTempFile, e); } + } catch (IOException e) { + LOGGER.warn("Could not delete temporary file " + libraryTempFile, e); } } } - @Override - protected void delete(IServletRequest request, IServletResponse response) { - Pair<DataverseName, String> libraryName = parseLibraryName(request); - if (libraryName == null) { - response.setStatus(HttpResponseStatus.BAD_REQUEST); - return; - } - try { - IRequestReference requestReference = receptionist.welcome(request); - doDrop(libraryName.first, libraryName.second, false, requestReference, request, response); - response.setStatus(HttpResponseStatus.OK); - } catch (Exception e) { - response.setStatus(toHttpErrorStatus(e)); - PrintWriter responseWriter = response.writer(); - responseWriter.write(e.getMessage()); - responseWriter.flush(); - LOGGER.error("Error deleting library " + libraryName.first + "." + libraryName.second, e); - } - } - } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfRecoveryServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfRecoveryServlet.java index 2c29d14..1563833 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfRecoveryServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfRecoveryServlet.java @@ -24,6 +24,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.asterix.common.api.IApplicationContext; import org.apache.asterix.common.api.INcApplicationContext; +import org.apache.asterix.compiler.provider.ILangCompilationProvider; import org.apache.asterix.external.library.ExternalLibraryManager; import org.apache.hyracks.http.api.IServletRequest; import org.apache.hyracks.http.api.IServletResponse; @@ -38,8 +39,8 @@ public class NCUdfRecoveryServlet extends AbstractNCUdfServlet { public static final String GET_ALL_UDF_ENDPOINT = "/all"; public NCUdfRecoveryServlet(ConcurrentMap<String, Object> ctx, String[] paths, IApplicationContext appCtx, - HttpScheme httpServerProtocol, int httpServerPort) { - super(ctx, paths, appCtx, httpServerProtocol, httpServerPort); + ILangCompilationProvider compilationProvider, HttpScheme httpServerProtocol, int httpServerPort) { + super(ctx, paths, appCtx, compilationProvider, httpServerProtocol, httpServerPort); } @Override diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java index b0cb4de..c27b3b1 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java @@ -228,9 +228,9 @@ public class NCApplication extends BaseNCApplication { new NCUdfApiServlet(apiServer.ctx(), new String[] { UDF }, getApplicationContext(), sqlppCompilationProvider, apiServer.getScheme(), apiServer.getAddress().getPort()), auth.getFirst(), auth.getSecond())); - apiServer.addServlet(new BasicAuthServlet( - apiServer.ctx(), new NCUdfRecoveryServlet(apiServer.ctx(), new String[] { UDF_RECOVERY }, - getApplicationContext(), apiServer.getScheme(), apiServer.getAddress().getPort()), + apiServer.addServlet(new BasicAuthServlet(apiServer.ctx(), + new NCUdfRecoveryServlet(apiServer.ctx(), new String[] { UDF_RECOVERY }, getApplicationContext(), + sqlppCompilationProvider, apiServer.getScheme(), apiServer.getAddress().getPort()), auth.getFirst(), auth.getSecond())); apiServer.addServlet(new QueryStatusApiServlet(apiServer.ctx(), getApplicationContext(), QUERY_STATUS)); apiServer.addServlet(new QueryResultApiServlet(apiServer.ctx(), getApplicationContext(), QUERY_RESULT)); diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java index 81e35a0..88277e4 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java @@ -23,8 +23,7 @@ import java.io.IOException; import java.net.URI; import org.apache.asterix.common.exceptions.AsterixException; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpEntity; +import org.apache.asterix.common.metadata.DataverseName; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.auth.AuthScope; @@ -32,7 +31,6 @@ import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.AuthCache; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.entity.ContentType; @@ -44,32 +42,58 @@ import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.hyracks.algebricks.common.utils.Pair; +import com.fasterxml.jackson.databind.ObjectMapper; + @SuppressWarnings("squid:S134") public class ExternalUDFLibrarian implements IExternalUDFLibrarian { private HttpClient hc; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + public ExternalUDFLibrarian() { hc = new DefaultHttpClient(); } @Override - public void install(URI path, String libPath, Pair<String, String> credentials) throws Exception { + public void install(URI path, String dataverseKey, DataverseName dataverse, boolean useDisplayForm, String name, + String type, String libPath, Pair<String, String> credentials) throws Exception { HttpClientContext hcCtx = createHttpClientContext(path, credentials); HttpPost post = new HttpPost(path); File lib = new File(libPath); - HttpEntity file = MultipartEntityBuilder.create().setMode(HttpMultipartMode.STRICT) - .addBinaryBody("lib", lib, ContentType.DEFAULT_BINARY, lib.getName()).build(); - post.setEntity(file); + MultipartEntityBuilder entity = MultipartEntityBuilder.create().setMode(HttpMultipartMode.STRICT); + if (!useDisplayForm) { + for (String dvPart : dataverse.getParts()) { + entity.addTextBody(dataverseKey, dvPart); + } + } else { + entity.addTextBody(dataverseKey, dataverse.toString()); + } + entity.addTextBody("name", name); + entity.addTextBody("type", type); + entity.addBinaryBody("data", lib, ContentType.DEFAULT_BINARY, lib.getName()).build(); + post.setEntity(entity.build()); HttpResponse response = hc.execute(post, hcCtx); handleResponse(response); } @Override - public void uninstall(URI path, Pair<String, String> credentials) throws IOException, AsterixException { + public void uninstall(URI path, String dataverseKey, DataverseName dataverse, boolean useDisplayForm, String name, + Pair<String, String> credentials) throws IOException, AsterixException { HttpClientContext hcCtx = createHttpClientContext(path, credentials); - HttpDelete del = new HttpDelete(path); - HttpResponse response = hc.execute(del, hcCtx); + HttpPost post = new HttpPost(path); + MultipartEntityBuilder entity = MultipartEntityBuilder.create().setMode(HttpMultipartMode.STRICT); + if (!useDisplayForm) { + for (String dvPart : dataverse.getParts()) { + entity.addTextBody(dataverseKey, dvPart); + } + } else { + entity.addTextBody(dataverseKey, dataverse.toString()); + } + entity.addTextBody("name", name); + entity.addTextBody("delete", "true"); + post.setEntity(entity.build()); + HttpResponse response = hc.execute(post, hcCtx); handleResponse(response); } @@ -89,7 +113,7 @@ public class ExternalUDFLibrarian implements IExternalUDFLibrarian { String resp = null; int respCode = response.getStatusLine().getStatusCode(); if (respCode == 500 || respCode == 400) { - resp = IOUtils.toString(response.getEntity().getContent()); + resp = OBJECT_MAPPER.readTree(response.getEntity().getContent()).get("error").asText(); } response.getEntity().consumeContent(); if (resp == null && respCode != 200) { diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java index 2315bec..639475b 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java @@ -22,11 +22,14 @@ import java.io.IOException; import java.net.URI; import org.apache.asterix.common.exceptions.AsterixException; +import org.apache.asterix.common.metadata.DataverseName; import org.apache.hyracks.algebricks.common.utils.Pair; public interface IExternalUDFLibrarian { - void install(URI path, String libPath, Pair<String, String> credentials) throws Exception; + void install(URI path, String dataverseKey, DataverseName dataverse, boolean useDisplayForm, String name, + String type, String libPath, Pair<String, String> credentials) throws Exception; - void uninstall(URI path, Pair<String, String> credentials) throws IOException, AsterixException; + void uninstall(URI path, String dataverseKey, DataverseName dataverse, boolean useDisplayForm, String name, + Pair<String, String> credentials) throws IOException, AsterixException; } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index 3cc6353..730ed46 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -82,6 +82,8 @@ import org.apache.asterix.common.api.Duration; import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.metadata.DataverseName; import org.apache.asterix.common.utils.Servlets; +import org.apache.asterix.lang.common.base.IParserFactory; +import org.apache.asterix.lang.sqlpp.parser.SqlppParserFactory; import org.apache.asterix.lang.sqlpp.util.SqlppStatementUtil; import org.apache.asterix.metadata.bootstrap.MetadataBuiltinEntities; import org.apache.asterix.metadata.utils.MetadataConstants; @@ -104,14 +106,25 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.AuthCache; +import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.apache.http.entity.mime.HttpMultipartMode; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import org.apache.http.impl.auth.BasicScheme; +import org.apache.http.impl.client.BasicAuthCache; +import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.StandardHttpRequestRetryHandler; @@ -173,6 +186,8 @@ public class TestExecutor { private static final Pattern VARIABLE_REF_PATTERN = Pattern.compile("\\$(\\w+)"); private static final Pattern HTTP_PARAM_PATTERN = Pattern.compile("param (?<name>[\\w-$]+)(?::(?<type>\\w+))?=(?<value>.*)", Pattern.MULTILINE); + private static final Pattern HTTP_AUTH_PATTERN = + Pattern.compile("auth (?<username>.*):(?<password>.*)", Pattern.MULTILINE); private static final Pattern HTTP_BODY_PATTERN = Pattern.compile("body=(.*)", Pattern.MULTILINE); private static final Pattern HTTP_STATUSCODE_PATTERN = Pattern.compile("statuscode (.*)", Pattern.MULTILINE); private static final Pattern MAX_RESULT_READS_PATTERN = @@ -619,6 +634,11 @@ public class TestExecutor { return checkResponse(executeHttpRequest(method), responseCodeValidator); } + protected HttpResponse executeAndCheckHttpRequest(HttpUriRequest method, Predicate<Integer> responseCodeValidator, + Pair<String, String> credentials) throws Exception { + return checkResponse(executeBasicAuthHttpRequest(method, credentials), responseCodeValidator); + } + protected HttpResponse executeHttpRequest(HttpUriRequest method) throws Exception { // https://issues.apache.org/jira/browse/ASTERIXDB-2315 ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -642,6 +662,36 @@ public class TestExecutor { } } + private HttpResponse executeBasicAuthHttpRequest(HttpUriRequest method, Pair<String, String> credentials) + throws Exception { + // https://issues.apache.org/jira/browse/ASTERIXDB-2315 + ExecutorService executor = Executors.newSingleThreadExecutor(); + CredentialsProvider cp = new BasicCredentialsProvider(); + cp.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(credentials.first, credentials.second)); + HttpClientContext hcCtx = HttpClientContext.create(); + AuthCache ac = new BasicAuthCache(); + ac.put(new HttpHost(method.getURI().getHost(), method.getURI().getPort(), "http"), new BasicScheme()); + hcCtx.setAuthCache(ac); + CloseableHttpClient client = HttpClients.custom().setRetryHandler(StandardHttpRequestRetryHandler.INSTANCE) + .setDefaultCredentialsProvider(cp).build(); + Future<HttpResponse> future = executor.submit(() -> { + try { + return client.execute(method, hcCtx); + } catch (Exception e) { + GlobalConfig.ASTERIX_LOGGER.log(Level.ERROR, "Failure executing {}", method, e); + throw e; + } + }); + try { + return future.get(); + } catch (Exception e) { + client.close(); + throw e; + } finally { + executor.shutdownNow(); + } + } + protected HttpContext getHttpContext() { return null; } @@ -867,14 +917,38 @@ public class TestExecutor { return builder.build(); } + private boolean isMultipart(Parameter p) { + return p != null && (ParameterTypeEnum.MULTIPART_TEXT == p.getType() + || ParameterTypeEnum.MULTIPART_BINARY == p.getType()); + } + + private void addMultipart(MultipartEntityBuilder multipartEntityBuilder, Parameter p) { + if (ParameterTypeEnum.MULTIPART_TEXT == p.getType()) { + multipartEntityBuilder.addTextBody(p.getName(), p.getValue()); + } else if (ParameterTypeEnum.MULTIPART_BINARY == p.getType()) { + File binary = new File(p.getValue()); + multipartEntityBuilder.addBinaryBody(p.getName(), binary, ContentType.DEFAULT_BINARY, binary.getName()); + } + } + private HttpUriRequest buildRequest(String method, URI uri, List<Parameter> params, Optional<String> body, ContentType contentType) { RequestBuilder builder = RequestBuilder.create(method); builder.setUri(uri); + Optional<MultipartEntityBuilder> mPartBuilder = params.stream() + .anyMatch(p -> p.getType() == ParameterTypeEnum.MULTIPART_BINARY + || p.getType() == ParameterTypeEnum.MULTIPART_TEXT) + ? Optional.of(MultipartEntityBuilder.create().setMode(HttpMultipartMode.STRICT)) + : Optional.empty(); for (Parameter param : params) { - builder.addParameter(param.getName(), param.getValue()); + if (isMultipart(param)) { + addMultipart(mPartBuilder.get(), param); + } else { + builder.addParameter(param.getName(), param.getValue()); + } } builder.setCharset(UTF_8); + mPartBuilder.ifPresent(mpb -> builder.setEntity(mpb.build())); body.ifPresent(s -> builder.setEntity(new StringEntity(s, contentType))); return builder.build(); } @@ -978,6 +1052,14 @@ public class TestExecutor { return response.getEntity().getContent(); } + private InputStream executeJSON(OutputFormat fmt, String method, URI uri, List<Parameter> params, + Predicate<Integer> responseCodeValidator, Optional<String> body, ContentType contentType, + Pair<String, String> credentials) throws Exception { + HttpUriRequest request = buildRequest(method, uri, fmt, params, body, contentType); + HttpResponse response = executeAndCheckHttpRequest(request, responseCodeValidator, credentials); + return response.getEntity().getContent(); + } + // Method that reads a DDL/Update/Query File // and returns the contents as a string // This string is later passed to REST API for execution. @@ -1239,30 +1321,37 @@ public class TestExecutor { // TODO: make this case work well with entity names containing spaces by // looking for \" lines = stripAllComments(statement).trim().split("\n"); + IParserFactory parserFactory = new SqlppParserFactory(); for (String line : lines) { String[] command = line.trim().split(" "); + URI path = createEndpointURI("/admin/udf/"); if (command.length < 2) { throw new Exception("invalid library command: " + line); } - String dataverse = command[1]; - String library = command[2]; - String username = command[3]; - String pw = command[4]; switch (command[0]) { case "install": - if (command.length != 6) { + if (command.length != 7) { throw new Exception("invalid library format"); } - String libPath = command[5]; - URI create = createEndpointURI("/admin/udf/" + dataverse + "/" + library); - librarian.install(create, libPath, new Pair<>(username, pw)); + List<String> dataverse = parserFactory.createParser(command[1]).parseMultipartIdentifier(); + String library = command[2]; + String type = command[3]; + String username = command[4]; + String pw = command[5]; + String libPath = command[6]; + librarian.install(path, "dataverse", DataverseName.create(dataverse), false, library, type, + libPath, new Pair<>(username, pw)); break; case "uninstall": if (command.length != 5) { throw new Exception("invalid library format"); } - URI delete = createEndpointURI("/admin/udf/" + dataverse + "/" + library); - librarian.uninstall(delete, new Pair<>(username, pw)); + dataverse = parserFactory.createParser(command[1]).parseMultipartIdentifier(); + library = command[2]; + username = command[3]; + pw = command[4]; + librarian.uninstall(path, "dataverse", DataverseName.create(dataverse), false, library, + new Pair<>(username, pw)); break; default: throw new Exception("invalid library format"); @@ -1382,9 +1471,16 @@ public class TestExecutor { if (!body.isPresent()) { body = getBodyFromReference(statement, variableCtx); } + final Pair<String, String> credentials = extractCredentials(statement); InputStream resultStream; if ("http".equals(extension)) { - resultStream = executeHttp(reqType, variablesReplaced, fmt, params, statusCodePredicate, body, contentType); + if (credentials != null) { + resultStream = executeHttp(reqType, variablesReplaced, fmt, params, statusCodePredicate, body, + contentType, credentials); + } else { + resultStream = + executeHttp(reqType, variablesReplaced, fmt, params, statusCodePredicate, body, contentType); + } } else if ("uri".equals(extension)) { resultStream = executeURI(reqType, URI.create(variablesReplaced), fmt, params, statusCodePredicate, body, contentType); @@ -1855,6 +1951,17 @@ public class TestExecutor { return params; } + public static Pair<String, String> extractCredentials(String statement) { + List<Parameter> params = new ArrayList<>(); + final Matcher m = HTTP_AUTH_PATTERN.matcher(statement); + while (m.find()) { + String username = m.group("username"); + String password = m.group("password"); + return new Pair<>(username, password); + } + return null; + } + private static String extractHttpRequestType(String statement) { Matcher m = HTTP_REQUEST_TYPE.matcher(statement); return m.find() ? m.group(1) : null; @@ -1904,6 +2011,13 @@ public class TestExecutor { return executeURI(ctxType, uri, fmt, params, statusCodePredicate, body, contentType); } + private InputStream executeHttp(String ctxType, String endpoint, OutputFormat fmt, List<Parameter> params, + Predicate<Integer> statusCodePredicate, Optional<String> body, ContentType contentType, + Pair<String, String> credentials) throws Exception { + URI uri = createEndpointURI(endpoint); + return executeURI(ctxType, uri, fmt, params, statusCodePredicate, body, contentType, credentials); + } + private InputStream executeURI(String ctxType, URI uri, OutputFormat fmt, List<Parameter> params) throws Exception { return executeJSON(fmt, ctxType.toUpperCase(), uri, params); } @@ -1913,6 +2027,13 @@ public class TestExecutor { return executeJSON(fmt, ctxType.toUpperCase(), uri, params, responseCodeValidator, body, contentType); } + private InputStream executeURI(String ctxType, URI uri, OutputFormat fmt, List<Parameter> params, + Predicate<Integer> responseCodeValidator, Optional<String> body, ContentType contentType, + Pair<String, String> credentials) throws Exception { + return executeJSON(fmt, ctxType.toUpperCase(), uri, params, responseCodeValidator, body, contentType, + credentials); + } + public void killNC(String nodeId, CompilationUnit cUnit) throws Exception { //get node process id OutputFormat fmt = OutputFormat.CLEAN_JSON; diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf b/asterixdb/asterix-app/src/test/resources/cc.conf index 1899122..e2cd5b9 100644 --- a/asterixdb/asterix-app/src/test/resources/cc.conf +++ b/asterixdb/asterix-app/src/test/resources/cc.conf @@ -33,6 +33,7 @@ nc.api.port=19005 [nc] credential.file=src/test/resources/security/passwd +python.cmd.autolocate=true address=127.0.0.1 command=asterixnc app.class=org.apache.asterix.hyracks.bootstrap.NCApplication diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/bad-ext-function-ddl-1/bad-ext-function-ddl-1.2.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/bad-ext-function-ddl-1/bad-ext-function-ddl-1.2.lib.sqlpp index 3dc6eb6..cc6c855 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/bad-ext-function-ddl-1/bad-ext-function-ddl-1.2.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/bad-ext-function-ddl-1/bad-ext-function-ddl-1.2.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip +install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/create-or-replace-function-1/create-or-replace-function-1.2.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/create-or-replace-function-1/create-or-replace-function-1.2.lib.sqlpp index 3dc6eb6..cc6c855 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/create-or-replace-function-1/create-or-replace-function-1.2.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/create-or-replace-function-1/create-or-replace-function-1.2.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip +install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.1.lib.sqlpp index 3dc6eb6..cc6c855 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.1.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip +install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/exception_create_system_library/exception_create_system_library.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/exception_create_system_library/exception_create_system_library.1.lib.sqlpp index ac3d3d0..1707699 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/exception_create_system_library/exception_create_system_library.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/exception_create_system_library/exception_create_system_library.1.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install Metadata testlib admin admin target/data/externallib/asterix-external-data-testlib.zip +install Metadata testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.1.lib.sqlpp index 3dc6eb6..cc6c855 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.1.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip +install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.1.lib.sqlpp index 3dc6eb6..cc6c855 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.1.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip +install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/invalid_library_requests/library_list_api_multipart.1.post.http similarity index 77% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/invalid_library_requests/library_list_api_multipart.1.post.http index 1650910..185d282 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/invalid_library_requests/library_list_api_multipart.1.post.http @@ -16,4 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/TweetSent.pyz +# auth admin:admin +# param dataverse:multipart_text=Default +# param name:multipart_text=testlib +# param type:multipart_text=badType +# param data:multipart_binary=target/data/externallib/asterix-external-data-testlib.zip + +/admin/udf diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/invalid_library_requests/library_list_api_multipart.2.post.http similarity index 75% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/invalid_library_requests/library_list_api_multipart.2.post.http index 1650910..01b4982 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/invalid_library_requests/library_list_api_multipart.2.post.http @@ -16,4 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/TweetSent.pyz +# auth admin:admin +# param dataverse:multipart_text=Default +# param name:multipart_text=testlib +# param type:multipart_text=java +# param delete:multipart_text=true +# param data:multipart_binary=target/data/externallib/asterix-external-data-testlib.zip + +/admin/udf diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/invalid_library_requests/library_list_api_multipart.3.post.http similarity index 80% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/invalid_library_requests/library_list_api_multipart.3.post.http index 1650910..01c05d2 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/invalid_library_requests/library_list_api_multipart.3.post.http @@ -16,4 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/TweetSent.pyz +# auth admin:admin +# param dataverse:multipart_text=Default +# param name:multipart_text=testlib +# param data:multipart_binary=target/data/externallib/asterix-external-data-testlib.zip + +/admin/udf diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/invalid_library_requests/library_list_api_multipart.4.post.http similarity index 82% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/invalid_library_requests/library_list_api_multipart.4.post.http index 1650910..0b6d882 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/invalid_library_requests/library_list_api_multipart.4.post.http @@ -16,4 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/TweetSent.pyz +# auth admin:admin +# param dataverse:multipart_text=Default +# param name:multipart_text=testlib +# param type:multipart_text=java +# param data:multipart_text=bogus + +/admin/udf diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/invalid_library_requests/library_list_api_multipart.5.post.http similarity index 85% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/invalid_library_requests/library_list_api_multipart.5.post.http index 1650910..e8de108 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/invalid_library_requests/library_list_api_multipart.5.post.http @@ -16,4 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/TweetSent.pyz +# auth admin:admin +# param dataverse:multipart_text=Default +# param name:multipart_text=testlib +# param type:multipart_text=java + +/admin/udf diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/keyword_detector/keyword_detector.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/keyword_detector/keyword_detector.1.lib.sqlpp index 63efff4..b1dba3d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/keyword_detector/keyword_detector.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/keyword_detector/keyword_detector.1.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install test testlib admin admin target/data/externallib/asterix-external-data-testlib.zip +install test testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api/library_list_api.0.ddl.sqlpp similarity index 91% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api/library_list_api.0.ddl.sqlpp index 0f0a05b..76cc70d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api/library_list_api.0.ddl.sqlpp @@ -16,5 +16,5 @@ * specific language governing permissions and limitations * under the License. */ - -install test testlib admin admin target/TweetSent.pyz +DROP DATAVERSE externallibtest if exists; +CREATE DATAVERSE externallibtest; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api/library_list_api.1.post.http similarity index 77% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api/library_list_api.1.post.http index 1650910..de72c49 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api/library_list_api.1.post.http @@ -16,4 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/TweetSent.pyz +# auth admin:admin +# param dataverse:multipart_text=externallibtest +# param name:multipart_text=testlib +# param type:multipart_text=java +# param data:multipart_binary=target/data/externallib/asterix-external-data-testlib.zip + +/admin/udf diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api/library_list_api.2.get.http similarity index 93% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api/library_list_api.2.get.http index 0f0a05b..5851101 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api/library_list_api.2.get.http @@ -16,5 +16,5 @@ * specific language governing permissions and limitations * under the License. */ - -install test testlib admin admin target/TweetSent.pyz +# auth admin:admin + /admin/udf diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/exception_create_system_library/exception_create_system_library.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.0.ddl.sqlpp similarity index 70% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/exception_create_system_library/exception_create_system_library.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.0.ddl.sqlpp index ac3d3d0..0c9da1e 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/exception_create_system_library/exception_create_system_library.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.0.ddl.sqlpp @@ -16,4 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -install Metadata testlib admin admin target/data/externallib/asterix-external-data-testlib.zip +DROP DATAVERSE externallibtest.foo if exists; +CREATE DATAVERSE externallibtest.foo; +DROP DATAVERSE externallibtest if exists; +CREATE DATAVERSE externallibtest; +DROP DATAVERSE `external`.lib.test if exists; +CREATE DATAVERSE `external`.lib.test; +DROP DATAVERSE externallibtest.foo.bar if exists; +CREATE DATAVERSE externallibtest.foo.bar; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.1.post.http similarity index 74% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.1.post.http index 1650910..c097ccc 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.1.post.http @@ -16,4 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/TweetSent.pyz +# auth admin:admin +# param dataverse:multipart_text=externallibtest +# param dataverse:multipart_text=foo +# param name:multipart_text=testlib +# param type:multipart_text=java +# param data:multipart_binary=target/data/externallib/asterix-external-data-testlib.zip + +/admin/udf diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.2.post.http similarity index 77% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.2.post.http index 1650910..de72c49 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.2.post.http @@ -16,4 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/TweetSent.pyz +# auth admin:admin +# param dataverse:multipart_text=externallibtest +# param name:multipart_text=testlib +# param type:multipart_text=java +# param data:multipart_binary=target/data/externallib/asterix-external-data-testlib.zip + +/admin/udf diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.3.post.http similarity index 72% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.3.post.http index 1650910..16b3596 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.3.post.http @@ -16,4 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/TweetSent.pyz +# auth admin:admin +# param dataverse:multipart_text=external +# param dataverse:multipart_text=lib +# param dataverse:multipart_text=test +# param name:multipart_text=testlib +# param type:multipart_text=java +# param data:multipart_binary=target/data/externallib/asterix-external-data-testlib.zip + +/admin/udf diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.4.post.http similarity index 72% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.4.post.http index 1650910..97a3c27 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.4.post.http @@ -16,4 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/TweetSent.pyz +# auth admin:admin +# param dataverse:multipart_text=externallibtest +# param dataverse:multipart_text=foo +# param dataverse:multipart_text=bar +# param name:multipart_text=testlib +# param type:multipart_text=java +# param data:multipart_binary=target/data/externallib/asterix-external-data-testlib.zip + +/admin/udf diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.5.get.http similarity index 93% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.5.get.http index 0f0a05b..5851101 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/library_list_api_multipart/library_list_api_multipart.5.get.http @@ -16,5 +16,5 @@ * specific language governing permissions and limitations * under the License. */ - -install test testlib admin admin target/TweetSent.pyz +# auth admin:admin + /admin/udf diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.1.lib.sqlpp index 9e53153..f2ddc74 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.1.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest2 testlib admin admin target/data/externallib/asterix-external-data-testlib.zip +install externallibtest2 testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp index 1650910..699e565 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/TweetSent.pyz +install externallibtest testlib python admin admin target/TweetSent.pyz diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.0.ddl.sqlpp similarity index 91% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.0.ddl.sqlpp index 0f0a05b..fe2ba4c 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.0.ddl.sqlpp @@ -16,5 +16,5 @@ * specific language governing permissions and limitations * under the License. */ - -install test testlib admin admin target/TweetSent.pyz +DROP DATAVERSE externallib.test if exists; +CREATE DATAVERSE externallib.test; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.1.lib.sqlpp similarity index 91% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.1.lib.sqlpp index 0f0a05b..67e22cb 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.1.lib.sqlpp @@ -16,5 +16,4 @@ * specific language governing permissions and limitations * under the License. */ - -install test testlib admin admin target/TweetSent.pyz +install externallib.test testlib python admin admin target/TweetSent.pyz diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.2.ddl.sqlpp similarity index 88% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.2.ddl.sqlpp index 1650910..eff35da 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.2.ddl.sqlpp @@ -16,4 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/TweetSent.pyz + USE externallib.test; + +create function sentiment(s) + as "sentiment", "TweetSent.sentiment" at testlib; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.3.query.sqlpp similarity index 93% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.3.query.sqlpp index 0f0a05b..dbfef4f 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.3.query.sqlpp @@ -16,5 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +use externallib.test; + +sentiment("bad"); -install test testlib admin admin target/TweetSent.pyz diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.4.query.sqlpp similarity index 93% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.4.query.sqlpp index 0f0a05b..d89c46d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.4.query.sqlpp @@ -16,5 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +use externallib.test; + +sentiment("great"); -install test testlib admin admin target/TweetSent.pyz diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.5.query.sqlpp similarity index 93% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.5.query.sqlpp index 0f0a05b..f46bde4 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.5.query.sqlpp @@ -16,5 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +use externallib.test; + +sentiment("meh"); -install test testlib admin admin target/TweetSent.pyz diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.6.ddl.sqlpp similarity index 93% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.6.ddl.sqlpp index 0f0a05b..ef74f58 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment_multipart/mysentiment_multipart.6.ddl.sqlpp @@ -16,5 +16,4 @@ * specific language governing permissions and limitations * under the License. */ - -install test testlib admin admin target/TweetSent.pyz +DROP DATAVERSE externallib.test; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.1.lib.sqlpp index 3dc6eb6..cc6c855 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.1.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip +install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_bad_credential/mysum_bad_credential.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_bad_credential/mysum_bad_credential.1.lib.sqlpp index c825a93..4f2f3b6 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_bad_credential/mysum_bad_credential.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_bad_credential/mysum_bad_credential.1.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin bad target/data/externallib/asterix-external-data-testlib.zip +install externallibtest testlib java admin bad target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_bad_credential/mysum_bad_credential.2.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_bad_credential/mysum_bad_credential.2.lib.sqlpp index 35d6ef8..8f2af66 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_bad_credential/mysum_bad_credential.2.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_bad_credential/mysum_bad_credential.2.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib root admin target/data/externallib/asterix-external-data-testlib.zip +install externallibtest testlib java root admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_bad_credential/mysum_bad_credential.3.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_bad_credential/mysum_bad_credential.3.lib.sqlpp index 3dc6eb6..cc6c855 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_bad_credential/mysum_bad_credential.3.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_bad_credential/mysum_bad_credential.3.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip +install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.1.lib.sqlpp index 3dc6eb6..cc6c855 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.1.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip +install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.1.lib.sqlpp index 0f0a05b..ef2c4e9 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.1.lib.sqlpp @@ -17,4 +17,4 @@ * under the License. */ -install test testlib admin admin target/TweetSent.pyz +install test testlib python admin admin target/TweetSent.pyz diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp index 0f0a05b..ef2c4e9 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_nested_access/py_nested_access.1.lib.sqlpp @@ -17,4 +17,4 @@ * under the License. */ -install test testlib admin admin target/TweetSent.pyz +install test testlib python admin admin target/TweetSent.pyz diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.1.lib.sqlpp index 1650910..699e565 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.1.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/TweetSent.pyz +install externallibtest testlib python admin admin target/TweetSent.pyz diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/return_invalid_type/return_invalid_type.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/return_invalid_type/return_invalid_type.1.lib.sqlpp index 3dc6eb6..cc6c855 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/return_invalid_type/return_invalid_type.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/return_invalid_type/return_invalid_type.1.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip +install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.1.lib.sqlpp index 6bbdd0e..b2bf929 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.1.lib.sqlpp @@ -17,4 +17,4 @@ * under the License. */ -install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip +install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.1.lib.sqlpp index b4197f1..009f699 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.1.lib.sqlpp @@ -16,5 +16,5 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip -install externallibtest2 testlib2 admin admin target/data/externallib/asterix-external-data-testlib.zip +install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip +install externallibtest2 testlib2 java admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.1.lib.sqlpp index 3dc6eb6..cc6c855 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.1.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip +install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/exception_create_system_adapter/exception_create_system_adapter.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/exception_create_system_adapter/exception_create_system_adapter.1.lib.sqlpp index 45cdfd1..fdb4118 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/exception_create_system_adapter/exception_create_system_adapter.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/exception_create_system_adapter/exception_create_system_adapter.1.lib.sqlpp @@ -21,4 +21,4 @@ * Expected Res : Success */ -install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip \ No newline at end of file +install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-adapter-cross-dv/feed-with-external-adapter-cross-dv.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-adapter-cross-dv/feed-with-external-adapter-cross-dv.1.lib.sqlpp index 806ad5e..76cd853 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-adapter-cross-dv/feed-with-external-adapter-cross-dv.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-adapter-cross-dv/feed-with-external-adapter-cross-dv.1.lib.sqlpp @@ -21,4 +21,4 @@ * Expected Res : Success */ -install externallibtest2 testlib admin admin target/data/externallib/asterix-external-data-testlib.zip \ No newline at end of file +install externallibtest2 testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-adapter/feed-with-external-adapter.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-adapter/feed-with-external-adapter.1.lib.sqlpp index 253c657..2370125 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-adapter/feed-with-external-adapter.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-adapter/feed-with-external-adapter.1.lib.sqlpp @@ -21,4 +21,4 @@ * Expected Res : Success */ -install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip \ No newline at end of file +install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.1.lib.sqlpp index f3ea94f..2b25300 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.1.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.1.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -install udfs testlib admin admin target/data/externallib/asterix-external-data-testlib.zip +install udfs testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/invalid-library-params/invalid-library-params.1.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/invalid-library-params/invalid-library-params.1.regexjson new file mode 100644 index 0000000..8cabc55 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/invalid-library-params/invalid-library-params.1.regexjson @@ -0,0 +1,3 @@ +{ + "error": "ASX0053: Parameters type and delete cannot be used together", +} \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api/library_list_ap1.1.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api/library_list_ap1.1.regexjson new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api/library_list_ap1.1.regexjson @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api/library_list_ap1.2.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api/library_list_ap1.2.regexjson new file mode 100644 index 0000000..9f289d1 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api/library_list_ap1.2.regexjson @@ -0,0 +1,5 @@ +[{ + "dataverse": "externallibtest", + "hash_md5": "R{[a-zA-Z0-9-]+}", + "name": "testlib" +}] diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_ap1.1.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_ap1.1.regexjson new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_ap1.1.regexjson @@ -0,0 +1 @@ +{} diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_ap1.2.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_ap1.2.regexjson new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_ap1.2.regexjson @@ -0,0 +1 @@ +{} diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_ap1.3.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_ap1.3.regexjson new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_ap1.3.regexjson @@ -0,0 +1 @@ +{} diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_ap1.4.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_ap1.4.regexjson new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_ap1.4.regexjson @@ -0,0 +1 @@ +{} diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_ap1.5.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_ap1.5.regexjson new file mode 100644 index 0000000..e5d039f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_ap1.5.regexjson @@ -0,0 +1,20 @@ +[{ + "dataverse": ["external", "lib", "test"], + "hash_md5": "R{[a-zA-Z0-9-]+}", + "name": "testlib" +}, +{ + "dataverse": "externallibtest", + "hash_md5": "R{[a-zA-Z0-9-]+}", + "name": "testlib" +}, +{ + "dataverse": ["externallibtest", "foo"], + "hash_md5": "R{[a-zA-Z0-9-]+}", + "name": "testlib" +}, +{ + "dataverse": ["externallibtest", "foo", "bar"], + "hash_md5": "R{[a-zA-Z0-9-]+}", + "name": "testlib" +}] diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml index e4669da..59dec11 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml @@ -29,6 +29,11 @@ </compilation-unit> </test-case> <test-case FilePath="external-library"> + <compilation-unit name="mysentiment_multipart"> + <output-dir compare="Text">mysentiment</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="external-library"> <compilation-unit name="python-fn-escape"> <output-dir compare="Text">python-fn-escape</output-dir> <expected-error>ImportError: Module was not found in library</expected-error> diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml index a9f52b4..28bbb98 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml @@ -62,6 +62,16 @@ </compilation-unit> </test-case> <test-case FilePath="external-library"> + <compilation-unit name="invalid_library_requests"> + <output-dir compare="Text">mysum_bad_credential</output-dir> + <expected-error>ASX3042: Unsupported function language badType</expected-error> + <expected-error>ASX1110: The parameters \"type\" and \"delete\" cannot be provided at the same time</expected-error> + <expected-error>ASX0049: Parameter(s) type or delete must be specified</expected-error> + <expected-error>ASX0047: Invalid value for parameter \"data\": Attribute</expected-error> + <expected-error>ASX0049: Parameter(s) data must be specified</expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="external-library"> <compilation-unit name="mysum_dropinuse"> <output-dir compare="Text">mysum_dropinuse</output-dir> <expected-error>ASX1148: Cannot drop library externallibtest.testlib being used by function externallibtest.mysum(2)</expected-error> @@ -73,6 +83,16 @@ </compilation-unit> </test-case> <test-case FilePath="external-library"> + <compilation-unit name="library_list_api"> + <output-dir compare="Text">library_list_api</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="external-library"> + <compilation-unit name="library_list_api_multipart"> + <output-dir compare="Text">library_list_api_multipart</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="external-library"> <compilation-unit name="getCapital"> <output-dir compare="Text">getCapital</output-dir> </compilation-unit> diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java index ef390f4..93fe92d 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java @@ -23,9 +23,11 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.security.MessageDigest; +import java.util.List; import org.apache.asterix.common.metadata.DataverseName; import org.apache.asterix.external.ipc.ExternalFunctionResultRouter; +import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.exceptions.HyracksException; import org.apache.hyracks.api.io.FileReference; @@ -33,6 +35,10 @@ import org.apache.hyracks.ipc.impl.IPCSystem; public interface ILibraryManager { + List<Pair<DataverseName, String>> getLibraryListing() throws IOException; + + String getLibraryHash(DataverseName dataverseName, String libraryName) throws IOException; + ILibrary getLibrary(DataverseName dataverseName, String libraryName) throws HyracksDataException; void closeLibrary(DataverseName dataverseName, String libraryName) throws HyracksDataException; diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/LibraryDescriptor.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/LibraryDescriptor.java index 6f128dc..3825d98 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/LibraryDescriptor.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/LibraryDescriptor.java @@ -31,14 +31,10 @@ import com.fasterxml.jackson.databind.node.ObjectNode; */ public class LibraryDescriptor implements IJsonSerializable { - private static final long serialVersionUID = 2L; + private static final long serialVersionUID = 3L; private static final String FIELD_LANGUAGE = "lang"; - private static final String FIELD_HASH = "hash"; - - public static final String FILE_EXT_ZIP = "zip"; - - public static final String FILE_EXT_PYZ = "pyz"; + public static final String FIELD_HASH = "hash_md5"; /** * The library's language diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index 50f6458..b9ebeb6 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -277,7 +277,7 @@ 3039 = Cannot parse list item of type %1$s 3040 = Argument type: %1$s 3041 = Unable to load/instantiate class %1$s -3042 = UDF of kind %1$s not supported +3042 = Unsupported function language %1$s 3043 = Unknown function kind %1$s 3044 = Library class loader already registered! 3045 = Cannot handle a function argument of type %1$s diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java index c5b9b53..dce5503 100755 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java @@ -21,6 +21,7 @@ package org.apache.asterix.external.library; import static com.fasterxml.jackson.databind.MapperFeature.SORT_PROPERTIES_ALPHABETICALLY; import static com.fasterxml.jackson.databind.SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS; +import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; @@ -41,9 +42,11 @@ import java.nio.file.attribute.BasicFileAttributes; import java.security.DigestOutputStream; import java.security.KeyStore; import java.security.MessageDigest; +import java.util.ArrayList; import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.zip.ZipEntry; @@ -51,17 +54,21 @@ import java.util.zip.ZipFile; import javax.net.ssl.SSLContext; +import org.apache.asterix.common.exceptions.AsterixException; +import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.common.functions.ExternalFunctionLanguage; import org.apache.asterix.common.library.ILibrary; import org.apache.asterix.common.library.ILibraryManager; import org.apache.asterix.common.library.LibraryDescriptor; import org.apache.asterix.common.metadata.DataverseName; +import org.apache.asterix.common.utils.StoragePathUtil; import org.apache.asterix.external.ipc.ExternalFunctionResultRouter; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpHeaders; import org.apache.http.HttpStatus; @@ -221,7 +228,7 @@ public final class ExternalLibraryManager implements ILibraryManager, ILifeCycle } private FileReference getDataverseDir(DataverseName dataverseName) throws HyracksDataException { - return getChildFileRef(storageDir, dataverseName.getCanonicalForm()); + return getChildFileRef(storageDir, StoragePathUtil.prepareDataverseName(dataverseName)); } @Override @@ -236,6 +243,68 @@ public final class ExternalLibraryManager implements ILibraryManager, ILifeCycle } @Override + public List<Pair<DataverseName, String>> getLibraryListing() throws IOException { + List<Pair<DataverseName, String>> libs = new ArrayList<>(); + Files.walkFileTree(storageDirPath, new SimpleFileVisitor<Path>() { + @Override + public FileVisitResult visitFile(Path currPath, BasicFileAttributes attrs) { + //never want to see any files + return FileVisitResult.TERMINATE; + } + + @Override + public FileVisitResult preVisitDirectory(Path currPath, BasicFileAttributes attrs) + throws HyracksDataException { + if (currPath.equals(storageDirPath) || currPath.getParent().equals(storageDirPath)) { + return FileVisitResult.CONTINUE; + } + if (currPath.getFileName().toString().codePointAt(0) == StoragePathUtil.DATAVERSE_CONTINUATION_MARKER) { + return FileVisitResult.CONTINUE; + } + final String candidateDvAndLib = storageDirPath.toAbsolutePath().normalize() + .relativize(currPath.toAbsolutePath().normalize()).toString(); + List<String> dvParts = new ArrayList<>(); + final String[] tokens = StringUtils.split(candidateDvAndLib, File.separatorChar); + if (tokens == null || tokens.length < 2) { + //? shouldn't happen + return FileVisitResult.TERMINATE; + } + //add first part, then all multiparts + dvParts.add(tokens[0]); + int currToken = 1; + for (; currToken < tokens.length && tokens[currToken] + .codePointAt(0) == StoragePathUtil.DATAVERSE_CONTINUATION_MARKER; currToken++) { + dvParts.add(tokens[currToken].substring(1)); + } + //we should only arrive at foo/^bar/^baz/.../^bat/lib + //anything else is fishy or empty + if (currToken != tokens.length - 1) { + return FileVisitResult.SKIP_SUBTREE; + } + String candidateLib = tokens[currToken]; + DataverseName candidateDv = DataverseName.create(dvParts); + FileReference candidateLibPath = findLibraryRevDir(candidateDv, candidateLib); + if (candidateLibPath != null) { + libs.add(new Pair<>(candidateDv, candidateLib)); + } + return FileVisitResult.SKIP_SUBTREE; + } + }); + return libs; + } + + @Override + public String getLibraryHash(DataverseName dataverseName, String libraryName) throws IOException { + FileReference revDir = findLibraryRevDir(dataverseName, libraryName); + if (revDir == null) { + throw HyracksDataException + .create(AsterixException.create(ErrorCode.EXTERNAL_UDF_EXCEPTION, "Library does not exist")); + } + LibraryDescriptor desc = getLibraryDescriptor(revDir); + return desc.getHash(); + } + + @Override public ILibrary getLibrary(DataverseName dataverseName, String libraryName) throws HyracksDataException { Pair<DataverseName, String> key = getKey(dataverseName, libraryName); synchronized (this) { @@ -545,9 +614,8 @@ public final class ExternalLibraryManager implements ILibraryManager, ILifeCycle outputFile.getFile().createNewFile(); IFileHandle fHandle = ioManager.open(outputFile, IIOManager.FileReadWriteMode.READ_WRITE, IIOManager.FileSyncMode.METADATA_ASYNC_DATA_ASYNC); - try { - WritableByteChannel outChannel = ioManager.newWritableChannel(fHandle); - OutputStream outputStream = Channels.newOutputStream(outChannel); + WritableByteChannel outChannel = ioManager.newWritableChannel(fHandle); + try (OutputStream outputStream = Channels.newOutputStream(outChannel)) { IOUtils.copyLarge(dataStream, outputStream, copyBuffer); outputStream.flush(); ioManager.sync(fHandle, true); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java index f9d30b1..1fa53ea 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java @@ -82,11 +82,18 @@ class ExternalScalarPythonFunctionEvaluator extends ExternalScalarFunctionEvalua super(finfo, args, argTypes, ctx); IApplicationConfig cfg = ctx.getServiceContext().getAppConfig(); String pythonPathCmd = cfg.getString(NCConfig.Option.PYTHON_CMD); + boolean findPython = cfg.getBoolean(NCConfig.Option.PYTHON_CMD_AUTOLOCATE); List<String> pythonArgs = new ArrayList<>(); if (pythonPathCmd == null) { - //if absolute path to interpreter is not specified, use environmental python - pythonPathCmd = "/usr/bin/env"; - pythonArgs.add("python3"); + //if absolute path to interpreter is not specified, try to use environmental python + if (findPython) { + pythonPathCmd = "/usr/bin/env"; + pythonArgs.add("python3"); + } else { + throw HyracksDataException.create(AsterixException.create(ErrorCode.EXTERNAL_UDF_EXCEPTION, + "Python interpreter not specified, and " + NCConfig.Option.PYTHON_CMD_AUTOLOCATE.ini() + + " is false")); + } } File pythonPath = new File(pythonPathCmd); List<String> sitePkgs = new ArrayList<>(); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployPrepareOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployPrepareOperatorDescriptor.java index 4f91b1a..c12bb58 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployPrepareOperatorDescriptor.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployPrepareOperatorDescriptor.java @@ -130,17 +130,9 @@ public class LibraryDeployPrepareOperatorDescriptor extends AbstractLibraryOpera switch (language) { case JAVA: - if (!LibraryDescriptor.FILE_EXT_ZIP.equals(fileExt)) { - // shouldn't happen - throw new IOException("Unexpected file type: " + fileExt); - } libraryManager.unzip(targetFile, contentsDir); break; case PYTHON: - if (!LibraryDescriptor.FILE_EXT_PYZ.equals(fileExt)) { - // shouldn't happen - throw new IOException("Unexpected file type: " + fileExt); - } boolean extractMsgPack = ctx.getJobletContext().getServiceContext().getAppConfig() .getBoolean(PYTHON_USE_BUNDLED_MSGPACK); shiv(targetFile, stageDir, contentsDir, extractMsgPack); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalLibraryUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalLibraryUtils.java index 6e3be21..9ab07eb 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalLibraryUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalLibraryUtils.java @@ -21,13 +21,18 @@ package org.apache.asterix.external.util; import java.io.IOException; import java.io.StringWriter; import java.security.MessageDigest; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import org.apache.asterix.common.library.ILibraryManager; +import org.apache.asterix.common.metadata.DataverseName; +import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.util.bytes.HexPrinter; public class ExternalLibraryUtils { private ExternalLibraryUtils() { - } public static String digestToHexString(MessageDigest digest) throws IOException { @@ -36,4 +41,15 @@ public class ExternalLibraryUtils { HexPrinter.printHexString(hashBytes, 0, hashBytes.length, hashBuilder); return hashBuilder.toString(); } + + public static Map<DataverseName, Map<String, String>> produceLibraryListing(ILibraryManager libraryManager) + throws IOException { + List<Pair<DataverseName, String>> libs = libraryManager.getLibraryListing(); + Map<DataverseName, Map<String, String>> dvToLibHashes = new TreeMap<>(); + for (Pair<DataverseName, String> lib : libs) { + dvToLibHashes.computeIfAbsent(lib.first, h -> new TreeMap<>()).put(lib.getSecond(), + libraryManager.getLibraryHash(lib.first, lib.second)); + } + return dvToLibHashes; + } } diff --git a/asterixdb/asterix-test-framework/src/main/resources/Catalog.xsd b/asterixdb/asterix-test-framework/src/main/resources/Catalog.xsd index 508236a..4ea9e0e 100644 --- a/asterixdb/asterix-test-framework/src/main/resources/Catalog.xsd +++ b/asterixdb/asterix-test-framework/src/main/resources/Catalog.xsd @@ -298,6 +298,8 @@ <xs:restriction base="xs:string"> <xs:enumeration value="string"/> <xs:enumeration value="json"/> + <xs:enumeration value="multipart_text"/> + <xs:enumeration value="multipart_binary"/> </xs:restriction> </xs:simpleType> diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java index d803732..22b240a 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java @@ -94,6 +94,7 @@ public class NCConfig extends ControllerConfig { IO_WORKERS_PER_PARTITION(POSITIVE_INTEGER, 2), IO_QUEUE_SIZE(POSITIVE_INTEGER, 10), PYTHON_CMD(STRING, (String) null), + PYTHON_CMD_AUTOLOCATE(BOOLEAN, false), PYTHON_ADDITIONAL_PACKAGES(STRING_ARRAY, new String[0]), PYTHON_USE_BUNDLED_MSGPACK(BOOLEAN, true), PYTHON_ARGS(STRING_ARRAY, (String[]) null), @@ -235,13 +236,15 @@ public class NCConfig extends ControllerConfig { case IO_QUEUE_SIZE: return "Length of the queue used for requests to write and read"; case PYTHON_CMD: - return "Absolute path to python interpreter. Defaults to environmental Python3"; + return "Absolute path to python interpreter"; case PYTHON_ADDITIONAL_PACKAGES: return "List of additional paths, separated by a path separator character, to add to sys.path behind msgpack and library package paths"; case PYTHON_USE_BUNDLED_MSGPACK: return "True to include bundled msgpack on Python sys.path, false to use system-provided msgpack"; case PYTHON_ARGS: return "Python args to pass to Python interpreter"; + case PYTHON_CMD_AUTOLOCATE: + return "Whether or not to attempt to automatically set PYTHON_CMD to a usable interpreter"; case CREDENTIAL_FILE: return "Path to HTTP basic credentials"; default:
