Xikui Wang has submitted this change and it was merged. Change subject: Add install/uninstall UDF to Ansible ......................................................................
Add install/uninstall UDF to Ansible 1. Add udf.sh for handling install/uninstall UDF packages to the cluster. 2. Remove unnecessary path check in TestLibrarian. Change-Id: If7ea9640a06e2b691c19a8a819307d84b55a679e Reviewed-on: https://asterix-gerrit.ics.uci.edu/1682 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Yingyi Bu <[email protected]> BAD: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java A asterixdb/asterix-server/src/main/opt/ansible/bin/udf.sh A asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy_udf.yml A asterixdb/asterix-server/src/main/opt/ansible/yaml/destroy_udf.yml 5 files changed, 155 insertions(+), 20 deletions(-) Approvals: Yingyi Bu: Looks good to me, approved Jenkins: Verified; No violations found; No violations found; Verified diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java index b4e1c75..5f86c28 100755 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java @@ -50,11 +50,11 @@ import org.apache.asterix.metadata.entities.Library; import org.apache.asterix.metadata.utils.MetadataUtil; import org.apache.asterix.runtime.formats.NonTaggedDataFormat; -import org.apache.hyracks.control.common.controllers.ControllerConfig; public class ExternalLibraryUtils { private static final Logger LOGGER = Logger.getLogger(ExternalLibraryUtils.class.getName()); + private static final FilenameFilter nonHiddenFileNameFilter = (dir, name) -> !name.startsWith("."); private ExternalLibraryUtils() { } @@ -72,19 +72,15 @@ // directory exists? if (installLibDir.exists()) { // get the list of files in the directory - for (String dataverse : installLibDir.list()) { - File dataverseDir = new File(installLibDir, dataverse); - String[] libraries = dataverseDir.list(); - for (String library : libraries) { + for (File dataverseDir : installLibDir.listFiles(File::isDirectory)) { + for (File libraryDir : dataverseDir.listFiles(File::isDirectory)) { // for each file (library), register library - registerLibrary(externalLibraryManager, dataverse, library); + registerLibrary(externalLibraryManager, dataverseDir.getName(), libraryDir.getName()); // is metadata node? if (isMetadataNode) { // get library file - File libraryDir = new File(installLibDir.getAbsolutePath() + File.separator + dataverse - + File.separator + library); // install if needed (i,e, add the functions, adapters, datasources, parsers to the metadata) - installLibraryIfNeeded(dataverse, libraryDir, uninstalledLibs); + installLibraryIfNeeded(dataverseDir.getName(), libraryDir, uninstalledLibs); } } } @@ -105,7 +101,7 @@ // directory exists? if (uninstallLibDir.exists()) { // list files - uninstallLibNames = uninstallLibDir.list(); + uninstallLibNames = uninstallLibDir.list(nonHiddenFileNameFilter); for (String uninstallLibName : uninstallLibNames) { // Get the <dataverse name - library name> pair String[] components = uninstallLibName.split("\\."); @@ -295,7 +291,6 @@ * * @param dataverse * @param libraryName - * @param installLibDir * @throws Exception */ protected static void registerLibrary(ILibraryManager externalLibraryManager, String dataverse, String libraryName) @@ -395,16 +390,26 @@ * @return the directory "$(ControllerConfig.defaultDir)/library": This needs to be improved */ protected static File getLibraryInstallDir() { - String workingDir = System.getProperty("user.dir"); - return new File(workingDir, "library"); + // Check managix directory first. If not exists, check app home. + File installDir = new File(System.getProperty("user.dir"), "library"); + if (!installDir.exists()) { + installDir = new File(System.getProperty("app.home", System.getProperty("user.home")) + + File.separator + "lib" + File.separator + "udfs"); + } + return installDir; } /** * @return the directory "$(ControllerConfig.defaultDir)/uninstall": This needs to be improved */ protected static File getLibraryUninstallDir() { - String workingDir = System.getProperty("user.dir"); - return new File(workingDir, "uninstall"); + // Check managix directory first. If not exists, check app home. + File uninstallDir = new File(System.getProperty("user.dir"), "uninstall"); + if (!uninstallDir.exists()) { + uninstallDir = new File(System.getProperty("app.home", System.getProperty("user.home")) + + File.separator + "lib" + File.separator + "udfs" + File.separator + "uninstall"); + } + return uninstallDir; } } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java index e92b5e8..c67d26c 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java @@ -35,8 +35,6 @@ public class TestLibrarian implements ITestLibrarian { - public static final String LIBRARY_DIR_NAME = "library"; - // The following list includes a library manager for the CC // and library managers for NCs (one-per-NC). private final List<ILibraryManager> libraryManagers; @@ -84,9 +82,6 @@ public static void removeLibraryDir() throws IOException { File installLibDir = ExternalLibraryUtils.getLibraryInstallDir(); - if (!installLibDir.getAbsolutePath().endsWith(LIBRARY_DIR_NAME)) { - throw new HyracksDataException("Invalid library directory"); - } FileUtils.deleteQuietly(installLibDir); } diff --git a/asterixdb/asterix-server/src/main/opt/ansible/bin/udf.sh b/asterixdb/asterix-server/src/main/opt/ansible/bin/udf.sh new file mode 100755 index 0000000..2f8b6fe --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/bin/udf.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +# Get options +usage() { echo "./udf.sh -m [i|u] -d DATAVERSE_NAME -l LIBRARY_NAME [-p UDF_PACKAGE_PATH]" 1>&2; exit 1; } + +while getopts ":d:l:p:m:" o; do + case "${o}" in + m) + mode=${OPTARG} + ;; + d) + dname=${OPTARG} + ;; + l) + libname=${OPTARG} + ;; + p) + tarpath=${OPTARG} + ;; + *) + echo $o + usage + ;; + esac +done +shift $((OPTIND-1)) + +if [[ -z $dname ]] || [[ -z $libname ]]; then + echo "Dataverse name or Library name is missing" +fi + +# Gets the absolute path so that the script can work no matter where it is invoked. +pushd `dirname $0` > /dev/null +SCRIPT_PATH=`pwd -P` +popd > /dev/null +ANSB_PATH=`dirname "${SCRIPT_PATH}"` + +INVENTORY=$ANSB_PATH/conf/inventory + +# Deploy/Destroy the UDF package on all nodes. +export ANSIBLE_HOST_KEY_CHECKING=false +if [[ $mode = "i" ]]; then + if [[ -z $tarpath ]]; then + echo "UDF package path is undefined" + else + echo "Install library to $dname.$libname" + ansible-playbook -i $INVENTORY $ANSB_PATH/yaml/deploy_udf.yml --extra-vars "dataverse=$dname libname=$libname package_path=$tarpath" + fi +elif [[ $mode == "u" ]]; then + echo "Uninstall library in $dname.$libname" + ansible-playbook -i $INVENTORY $ANSB_PATH/yaml/destroy_udf.yml --extra-vars "dataverse=$dname libname=$libname" +else + echo "Wrong mode" +fi \ No newline at end of file diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy_udf.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy_udf.yml new file mode 100644 index 0000000..e9300ea --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy_udf.yml @@ -0,0 +1,30 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- name: Deploy the UDF library {{ dataverse }}.{{ libname }} to the cluster + hosts: all + tasks: + - include_vars: ../conf/instance_settings.yml + - file: + path: "{{ binarydir }}/lib/udfs/{{ dataverse }}/{{ libname }}" + state: directory + mode: 0755 + - unarchive: + src: "{{ package_path }}" + dest: "{{ binarydir }}/lib/udfs/{{ dataverse }}/{{ libname }}" diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/destroy_udf.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/destroy_udf.yml new file mode 100644 index 0000000..cc4add3 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/destroy_udf.yml @@ -0,0 +1,33 @@ +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +- name: Destroy the library {{ libname }} in dataverse {{ dataverse }} + hosts: all + tasks: + - include_vars: ../conf/instance_settings.yml + - file: + path: "{{ binarydir }}/lib/udfs/uninstall/" + state: directory + mode: 0755 + - file: + state: touch + path: "{{ binarydir }}/lib/udfs/uninstall/{{ dataverse }}.{{ libname }}" + - file: + path: "{{ binarydir }}/lib/udfs/{{ dataverse }}/{{ libname }}" + state: absent -- To view, visit https://asterix-gerrit.ics.uci.edu/1682 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: If7ea9640a06e2b691c19a8a819307d84b55a679e Gerrit-PatchSet: 3 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Xikui Wang <[email protected]> Gerrit-Reviewer: Yingyi Bu <[email protected]>
