This is an automated email from the ASF dual-hosted git repository. wiener pushed a commit to branch dev in repository https://gitbox.apache.org/repos/asf/incubator-streampipes-installer.git
commit 4aa21d3fd741a4eafc1fa6e242f22a71a55e9e0a Author: Patrick Wiener <[email protected]> AuthorDate: Thu Apr 8 17:08:29 2021 +0200 [STREAMPIPES-341] new add command to easily integrate developed pipeline elements --- cli/README.md | 32 ++++++++ cli/bin/commands/add | 212 +++++++++++++++++++++++++++++++++++++++++++++++++++ cli/streampipes | 4 + 3 files changed, 248 insertions(+) diff --git a/cli/README.md b/cli/README.md index 1d03684..0cfde35 100644 --- a/cli/README.md +++ b/cli/README.md @@ -64,6 +64,7 @@ Options: --version, -v show version Commands: + add Add new StreamPipes service (e.g., adapter, processor, sink) clean Remove StreamPipes data volumes, dangling images and network down Stop and remove StreamPipes containers env Inspect and select StreamPipes environments @@ -174,6 +175,37 @@ streampipes clean # streampipes clean --volumes ``` +## Add newly developed StreamPipes Pipeline Elements to catalog +As you develop new pipeline elements, e.g., adapters, processors or sinks, you might want to add them to your existing environment. Adding these components requires minimal effort. +We can simply use `streampipes add` command to populate relevant Docker Compose files under the StreamPipes working directory (`/path/to/cli/deploy/standalone/`). + +> **NOTE**: This requires your own pipeline elements to be already containerized and accessible to pull, e.g., from a public Docker repository such as DockerHub. If you use a private +> repository make sure your local Docker Daemon has access to it. + +```bash +# check out streampipes add --help for more options +# add new pipeline element to catalog, here my-processor, optional image=my-processo-image:0.67.0, ports 8090:8090 (external:internal) +streampipes add my-processor --image my-processor-image:0.67.0 --ports 8090:8090 +[INFO] Create new directory for: /path/to/cli/deploy/standalone/my-processor +[INFO] Generate Docker Compose files for my-processor +[INFO] Add service to current environment: lite ? +Add [y/n]: y +[INFO] Added service to .spenv +``` + +This creates necessary Docker Compose files in the local catalog: +```bash +cli/deploy/standalone/my-processor +├── docker-compose.dev.yml +└── docker-compose.yml +``` +Additionally, you were asked wether you want to add the new service to your current environment file `.spenv` (see `streampipes env`). + +You can now directly start you new pipeline element using the following command: +```bash +streampipes up -d my-processor +``` + ## Modify/Create an environment template As of now, this step has to be done **manually**. All environments are located in `environments/`. diff --git a/cli/bin/commands/add b/cli/bin/commands/add new file mode 100755 index 0000000..e667c32 --- /dev/null +++ b/cli/bin/commands/add @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +. "$STREAMPIPES_WORKDIR/bin/common" + +docker_compose_api_version=3.4 +svc_has_ports=false + +cli_help_start() { + cat <<EOF +Add new StreamPipes service to catalog. + +Usage: streampipes add SERVICE [OPTIONS] + +Examples: +streampipes add my-service +streampipes add my-service --image apachestreampipes/my-service:0.68.0 --ports 8090:8090 --ports 8091:8091 + +Options: + -i, --image Name of image (Default: ${SP_DOCKER_REGISTRY}/my-service:${SP_VERSION}) + -p, --ports Port mapping for this service (externalPort:internalPort) +EOF + + exit 1 +} + +[ "$1" == '--help' ] || [ "$1" == '-h' ] && cli_help_start + +if [ -z "$1" ]; then + fatal "Argument missing, see 'streampipes ${0##*/} --help'" +fi + +while [[ "$#" -gt 0 ]]; do + case $1 in + -i|--image) + if [ -n "$2" ] && [ ${2:0:1} != "-" ]; then + svc_img_tag="$2" + shift 2 + else + fatal "StreamPipes service image name for $1 is missing" >&2 + fi + ;; + -p|--ports) + if [ -n "$2" ] && [ ${2:0:1} != "-" ]; then + ports+=("$2") + svc_has_ports=true + shift 2 + else + fatal "Ports for $1 is missing" >&2 + fi + ;; + -*|--*=) fatal "Unsupported flag $1, see 'streampipes ${0##*/} --help'" >&2 ; shift ;; + # *) fatal "Wrong usage, see 'streampipes ${0##*/} --help'" ;; + *) + if [ ! -z "$1" ]; then + svc_name=$1 + shift + else + fatal "Wrong usage, see 'streampipes ${0##*/} --help'" + fi + ;; + esac +done + +apache_license_header(){ + cat << EOF +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +EOF +} + + +add_docker_compose_file() { + cat << EOF +$(apache_license_header) + +version: "${docker_compose_api_version}" +services: + ${svc_name}: + image: ${svc_img_tag} + depends_on: + - "consul" + - "backend" + logging: + driver: "json-file" + options: + max-size: "1m" + max-file: "1" + networks: + spnet: + +networks: + spnet: + external: true +EOF +} + +add_docker_compose_dev_file_with_ports() { + cat << EOF +$(apache_license_header) + +version: "${docker_compose_api_version}" +services: + ${svc_name}: +EOF + cat << EOF + ports: +$(get_port_mappings) +EOF +} + +add_docker_compose_dev_file_without_ports() { + cat << EOF +$(apache_license_header) + +version: "${docker_compose_api_version}" +services: + ${svc_name}: +EOF +} + +get_port_mappings(){ + for port_mapping in "${ports[@]}" + do + cat << EOF + - "$port_mapping" +EOF + done +} + +create_dir(){ + if [ ! -d "$1" ]; then + mkdir $1 + info "Create new directory for: $1" + else + warning "Service already exists $svc_name" + read -p 'Continue [y/n]: ' continue + if [ "$continue" == "n" ] || [ "$continue" == "n" ]; then + exit 1 + fi + fi +} + +check_img_or_default(){ + if [ -z "$svc_img_tag" ]; then + svc_img_tag="\${SP_DOCKER_REGISTRY}/$svc_name:\${SP_VERSION}" + fi +} + +create_docker_compose_files(){ + info "Generate Docker Compose files for $svc_name" + check_img_or_default + add_docker_compose_file > $1/docker-compose.yml + + if $svc_has_ports; then + add_docker_compose_dev_file_with_ports > $1/docker-compose.dev.yml + else + add_docker_compose_dev_file_without_ports > $1/docker-compose.dev.yml + fi +} + +append_to_curr_env(){ + info "Added service to .spenv" + curr_env_file=$STREAMPIPES_WORKDIR/.spenv + grep -qxF $svc_name $curr_env_file || echo $svc_name >> $curr_env_file +} + +add_to_current_environment(){ + get_curr_environment + info "Add service to current environment: $curr_environment ?" + read -p "Add [y/n]: " add + if [ "$add" == "n" ] || [ "$add" == "n" ]; then + exit 1 + else + append_to_curr_env + fi +} + +add_service() { + svc_dir=$STREAMPIPES_WORKDIR/deploy/standalone/${svc_name} + create_dir $svc_dir + create_docker_compose_files $svc_dir + add_to_current_environment +} + +add_service \ No newline at end of file diff --git a/cli/streampipes b/cli/streampipes index fd75255..cf85fa5 100755 --- a/cli/streampipes +++ b/cli/streampipes @@ -29,6 +29,7 @@ Options: --version, -v show version Commands: + add Add new StreamPipes service (e.g., adapter, processor, sink) clean Remove StreamPipes data volumes, dangling images and network down Stop and remove StreamPipes containers env Inspect and select StreamPipes environments @@ -48,6 +49,9 @@ EOF } case "$1" in + add) + "$STREAMPIPES_WORKDIR/bin/commands/add" "${@:2}" + ;; clean) "$STREAMPIPES_WORKDIR/bin/commands/clean" "${@:2}" ;;
