This is an automated email from the ASF dual-hosted git repository. xiazcy pushed a commit to branch 3.5-dev in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
The following commit(s) were added to refs/heads/3.5-dev by this push: new 0dc2522284 Update docker/build.sh to work with GLV docker-compose changes. new 933a01009c Merge pull request #1800 from Bit-Quill/ken/docker-build-sh 0dc2522284 is described below commit 0dc2522284584083680956cd5ab4480bd93468ec Author: Ken Hu <106191785+kenh...@users.noreply.github.com> AuthorDate: Thu Sep 1 17:42:39 2022 -0700 Update docker/build.sh to work with GLV docker-compose changes. The non-Java GLVs have changed to use docker-compose for integration testing which creates a Docker in Docker issue when run through the build.sh script. To prevent this issue, the GLVs are split away and run individually using their docker-compose files. However, since the docker-compose files require some artifacts from the build (including the local Maven repository items), those directories are bind mounted so that the files are available for the docker-compose builds. To increase the usability of the script for developers who only work on a portion of TinkerPop, options were added to include only a subset of the modules to the build. Change version parsing to use awk instead of grep with Perl regex because it isn't installed by default on macOS. --- docker/build.sh | 133 +++++++++++++++++++-- docker/scripts/build.sh | 59 +++++---- .../dev/developer/development-environment.asciidoc | 15 ++- gremlin-go/run.sh | 2 +- 4 files changed, 173 insertions(+), 36 deletions(-) diff --git a/docker/build.sh b/docker/build.sh index 3232ce5b5c..1b299d5c34 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -28,17 +28,72 @@ function cleanup { BUILD_IMAGE=$(docker images tinkerpop | awk "{if (\$2 == \"${BUILD_TAG}\") print \$3}") [ ! -z ${BUILD_IMAGE} ] && docker rmi ${BUILD_IMAGE} rm -f ${PROJECT_HOME}/Dockerfile + docker image prune --filter "label=maintainer=d...@tinkerpop.apache.org" -f > /dev/null } trap cleanup EXIT REMOVE_CONTAINER="--rm" [[ -n ${KEEP_CONTAINER} ]] && unset REMOVE_CONTAINER -case "$1" in - -h | --help ) ${DIR}/scripts/build.sh -h; exit 0 ;; -esac +function usage { + echo -e "\nUsage: `basename $0` [OPTIONS]" \ + "\nBuild the current local TinkerPop project in a Docker container." \ + "\nBy default all modules are run. If you prefer to run a specific module then please" \ + "\nenter it as an option. Selecting specific modules changes the build to include only" \ + "\nthose selected modules as well as gremlin-server, gremlin-test and neo4j-gremlin (and" \ + "\ntheir dependencies)." \ + "\n\nOptions are:\n" \ + "\n\t-t, --tests run standard test suite" \ + "\n\t-i, --integration-tests run integration tests" \ + "\n\t-n, --neo4j include Neo4j" \ + "\n\t-go, --golang change to minimal build and add gremlin-go to build" \ + "\n\t-py, --python change to minimal build and add gremlin-python to build" \ + "\n\t-dn, --dotnet change to minimal build and add gremlin-dotnet to build" \ + "\n\t-js, --javascript change to minimal build and add gremlin-javascript to build" \ + "\n\t-c, --console change to minimal build and add gremlin-console to build" \ + "\n\t-j, --java-docs build Java docs" \ + "\n\t-d, --docs build user docs" \ + "\n\t-h, --help show this message" \ + "\n" +} + +ARGS="" +RUN_TESTS="" +RUN_INTEGRATION_TESTS="" +INCLUDE_GO="" +INCLUDE_PYTHON="" +INCLUDE_DOTNET="" +INCLUDE_JAVASCRIPT="" +INCLUDE_CONSOLE="" +while [ ! -z "$1" ]; do + case "$1" in + -t | --tests ) ARGS="${ARGS} -t"; RUN_TESTS=true; shift ;; + -i | --integration-tests ) ARGS="${ARGS} -i"; RUN_INTEGRATION_TESTS=true; shift ;; + -n | --neo4j ) ARGS="${ARGS} -n"; shift ;; + -go | --golang ) ARGS="${ARGS} -go"; INCLUDE_GO=true; shift ;; + -py | --python ) ARGS="${ARGS} -py"; INCLUDE_PYTHON=true; shift ;; + -dn | --dotnet ) ARGS="${ARGS} -dn"; INCLUDE_DOTNET=true; shift ;; + -js | --javascript ) ARGS="${ARGS} -js"; INCLUDE_JAVASCRIPT=true; shift ;; + -c | --console ) ARGS="${ARGS} -c"; INCLUDE_CONSOLE=true; shift ;; + -j | --java-docs ) ARGS="${ARGS} -j"; shift ;; + -d | --docs ) ARGS="${ARGS} -d"; shift ;; + -h | --help ) usage; exit 0 ;; + *) usage 1>&2; exit 1 ;; + esac +done + +# The default is for every module to be included. +if [[ -z ${INCLUDE_GO} && -z ${INCLUDE_PYTHON} && -z ${INCLUDE_DOTNET} && -z ${INCLUDE_JAVASCRIPT} && -z ${INCLUDE_CONSOLE} ]]; then + INCLUDE_GO=true + INCLUDE_PYTHON=true + INCLUDE_DOTNET=true + INCLUDE_JAVASCRIPT=true + INCLUDE_CONSOLE=true +fi pushd ${PROJECT_HOME} > /dev/null +export ABS_PROJECT_HOME=$(pwd) # absolute path required by some Docker variables. +echo "ABS_PROJECT_HOME ${ABS_PROJECT_HOME}" HADOOP_VERSION=$(cat pom.xml | grep -o '<hadoop.version>[^<]*' | head -n1 | cut -d '>' -f2) @@ -46,12 +101,74 @@ docker/build-containers.sh -h "${HADOOP_VERSION}" sed -e "s/HADOOP_VERSION\$/${HADOOP_VERSION}/" docker/build/Dockerfile.template > Dockerfile cat >> Dockerfile <<EOF -CMD ["sh", "-c", "docker/scripts/build.sh $@"] +CMD ["sh", "-c", "docker/scripts/build.sh ${ARGS}"] EOF +function check_status { + status=$? + [ "$1" == "down" ] && docker-compose down + popd > /dev/null + [ $status -ne 0 ] && exit $status +} + +# GREMLIN_SERVER is the project version e.g. 3.5.5-SNAPSHOT +export GREMLIN_SERVER=$(grep tinkerpop -A2 pom.xml | sed -r -n 's/.*<version>(([0-9]+\.?){3})(-SNAPSHOT)?<\/version>/\1\3/p') +echo "GREMLIN_SERVER ${GREMLIN_SERVER}" + docker build -t tinkerpop:${BUILD_TAG} . -docker run -p 80:80 ${TINKERPOP_DOCKER_OPTS} ${REMOVE_CONTAINER} -e "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" -ti tinkerpop:${BUILD_TAG} +docker run -p 80:80 ${TINKERPOP_DOCKER_OPTS} ${REMOVE_CONTAINER} \ + -e "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" \ + -ti \ + --mount type=bind,src=${HOME}/.m2/,dst=/root/.m2/ \ + --mount type=bind,src=$(pwd)/gremlin-server/,dst=/usr/src/tinkerpop/gremlin-server/ \ + --mount type=bind,src=$(pwd)/gremlin-test/,dst=/usr/src/tinkerpop/gremlin-test/ \ + --mount type=bind,src=$(pwd)/gremlin-console/,dst=/usr/src/tinkerpop/gremlin-console/ \ + --mount type=bind,src=$(pwd)/neo4j-gremlin/,dst=/usr/src/tinkerpop/neo4j-gremlin/ \ + tinkerpop:${BUILD_TAG} +check_status + +if [ -n "${RUN_TESTS}" ]; then + # If testing, then build base server which is required by the following docker-compose. + pushd ${ABS_PROJECT_HOME}/gremlin-server > /dev/null + docker build -f ./Dockerfile --build-arg GREMLIN_SERVER_DIR=target/apache-tinkerpop-gremlin-server-${GREMLIN_SERVER}-standalone -t tinkerpop/gremlin-server:${GREMLIN_SERVER} . + check_status +fi + +if [ -n "${INCLUDE_GO}" ] && [ -n "${RUN_TESTS}" ]; then + pushd ${ABS_PROJECT_HOME}/gremlin-go > /dev/null + docker-compose up --build --exit-code-from gremlin-go-integration-tests + check_status "down" +fi + +if [ -n "${INCLUDE_PYTHON}" ] && [ -n "${RUN_TESTS}" ]; then + pushd ${ABS_PROJECT_HOME}/gremlin-python > /dev/null + export BUILD_DIR=$(pwd)/target/python3/ + mkdir -p ${BUILD_DIR} + cp -r ./src/main/python/* ${BUILD_DIR} + docker-compose up --build --abort-on-container-exit gremlin-server-test-python gremlin-python-integration-tests + check_status "down" +fi + +if [ -n "${INCLUDE_DOTNET}" ] && [ -n "${RUN_TESTS}" ]; then + pushd ${ABS_PROJECT_HOME}/gremlin-dotnet/test > /dev/null + docker-compose up --build --exit-code-from gremlin-dotnet-integration-tests + check_status "down" +fi + +if [ -n "${INCLUDE_JAVASCRIPT}" ] && [ -n "${RUN_TESTS}" ]; then + pushd ${ABS_PROJECT_HOME}/gremlin-javascript/src/main/javascript/gremlin-javascript > /dev/null + docker-compose up --build --exit-code-from gremlin-js-integration-tests + check_status "down" +fi + +if [ -n "${INCLUDE_CONSOLE}" ] && [ -n "${RUN_INTEGRATION_TESTS}" ]; then + pushd ${ABS_PROJECT_HOME}/gremlin-console > /dev/null + docker build -t gremlin-console-test:py3.8jre11 ./src/test/python/docker + docker run --rm \ + --mount type=bind,src=$(pwd)/src/test/python,dst=/console_app \ + --mount type=bind,src=$(pwd)/target/apache-tinkerpop-gremlin-console-${GREMLIN_SERVER}-standalone,dst=/console_app/gremlin-console \ + gremlin-console-test:py3.8jre11 + check_status +fi -status=$? -popd > /dev/null -exit $status +exit 0 diff --git a/docker/scripts/build.sh b/docker/scripts/build.sh index ee32fe70ec..37f948b733 100755 --- a/docker/scripts/build.sh +++ b/docker/scripts/build.sh @@ -23,39 +23,51 @@ RUN_INTEGRATION_TESTS= INCLUDE_NEO4J= BUILD_JAVA_DOCS= BUILD_USER_DOCS= - -function usage { - echo -e "\nUsage: `basename $0` [OPTIONS]" \ - "\nBuild the current local TinkerPop project in a Docker container." \ - "\n\nOptions are:\n" \ - "\n\t-t, --tests run standard test suite" \ - "\n\t-i, --integration-tests run integration tests" \ - "\n\t-n, --neo4j include Neo4j" \ - "\n\t-j, --java-docs build Java docs" \ - "\n\t-d, --docs build user docs" \ - "\n\t-h, --help show this message" \ - "\n" -} +INCLUDE_GO= +INCLUDE_PYTHON= +INCLUDE_DOTNET= +INCLUDE_JAVASCRIPT= +INCLUDE_CONSOLE= while [ ! -z "$1" ]; do case "$1" in - -t | --tests ) RUN_TESTS=true; shift ;; - -i | --integration-tests ) RUN_INTEGRATION_TESTS=true; shift ;; - -n | --neo4j ) INCLUDE_NEO4J=true; shift ;; - -j | --java-docs ) BUILD_JAVA_DOCS=true; shift ;; - -d | --docs ) BUILD_USER_DOCS=true; shift ;; - -h | --help ) usage; exit 0 ;; - *) usage 1>&2; exit 1 ;; + -t | --tests ) RUN_TESTS=true; shift ;; + -i | --integration-tests ) RUN_INTEGRATION_TESTS=true; shift ;; + -n | --neo4j ) INCLUDE_NEO4J=true; shift ;; + -j | --java-docs ) BUILD_JAVA_DOCS=true; shift ;; + -d | --docs ) BUILD_USER_DOCS=true; shift ;; + -go | --golang ) INCLUDE_GO=true; shift ;; + -py | --python ) INCLUDE_PYTHON=true; shift ;; + -dn | --dotnet ) INCLUDE_DOTNET=true; shift ;; + -js | --javascript ) INCLUDE_JAVASCRIPT=true; shift ;; + -c | --console ) INCLUDE_CONSOLE=true; shift ;; esac done -TINKERPOP_BUILD_OPTIONS="" +# To prevent Docker in Docker, skip building images and deactivate Docker profiles. +TINKERPOP_BUILD_OPTIONS="-DskipImageBuild -P -glv-js,-glv-go,-glv-python" [ -z "${RUN_TESTS}" ] && TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS} -DskipTests" [ -z "${RUN_INTEGRATION_TESTS}" ] || TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS} -DskipIntegrationTests=false" [ -z "${INCLUDE_NEO4J}" ] || TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS} -DincludeNeo4j" [ -z "${BUILD_JAVA_DOCS}" ] && TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS} -Dmaven.javadoc.skip=true" +# If any of these GLVs are selected then create a minimal build to reduce build/test times +# as the user is likely not interested in the other modules. +if [[ -n ${INCLUDE_GO} || -n ${INCLUDE_PYTHON} || -n ${INCLUDE_DOTNET} || -n ${INCLUDE_JAVASCRIPT} || -n ${INCLUDE_CONSOLE} ]]; then + # gremlin-server, gremlin-test and neo4j-gremlin are minimal dependencies needed to start a server for GLV testing. + TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS} -am -pl gremlin-server,gremlin-test,neo4j-gremlin" + + [ -n "${INCLUDE_GO}" ] && TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS},gremlin-go" + [ -n "${INCLUDE_PYTHON}" ] && TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS},gremlin-python" + [ -n "${INCLUDE_DOTNET}" ] && TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS},gremlin-dotnet,:gremlin-dotnet-source" + [ -n "${INCLUDE_JAVASCRIPT}" ] && TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS},gremlin-javascript" + [ -n "${INCLUDE_CONSOLE}" ] && TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS},gremlin-console" +else + # When building everything, deactivate dotnet tests to prevent Docker in Docker. + TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS} -pl -:gremlin-dotnet-tests" +fi + # If the tmpfs (in-memory filesystem exists, use it) TINKERMEM_PATH=$(cd .. ; echo `pwd`/tinkermem) if [ -d "${TINKERMEM_PATH}" ]; then @@ -67,9 +79,7 @@ if [ -d "${TINKERMEM_PATH}" ]; then cd ${TINKERMEM_PATH} fi -# omitting gremlin-go for now to avoid docker in docker as it builds via docker compose -# progress to restructure and add gremlin-go back will be tracked here https://issues.apache.org/jira/browse/TINKERPOP-2737 -touch {gremlin-dotnet,gremlin-dotnet/src,gremlin-dotnet/test,gremlin-python,gremlin-javascript}/.glv +touch gremlin-dotnet/src/.glv # use a custom maven settings.xml if [ -r "settings.xml" ]; then @@ -80,6 +90,7 @@ fi export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 mvn clean install process-resources --batch-mode ${TINKERPOP_BUILD_OPTIONS} || exit 1 + [ -z "${BUILD_JAVA_DOCS}" ] || mvn process-resources -Djavadoc || exit 1 if [ ! -z "${BUILD_USER_DOCS}" ]; then diff --git a/docs/src/dev/developer/development-environment.asciidoc b/docs/src/dev/developer/development-environment.asciidoc index 3ac74dbd4d..6b7797647a 100644 --- a/docs/src/dev/developer/development-environment.asciidoc +++ b/docs/src/dev/developer/development-environment.asciidoc @@ -448,9 +448,6 @@ TinkerPop provides a shell script, that can start several build tasks within a D required Docker images will be built automatically if they don't exist yet. Thus the first invocation of the Docker script is expected to take some time. -NOTE: Currently, `gremlin-go` integration is not supported within this Docker container, but can be run separately with Docker. -Please see the `Testing With Docker` section under the xref:../../../../gremlin-go/driver/README.md[Gremlin Go Driver] for details. - The script can be found under `PROJECT_HOME/docker/build.sh`. The following tasks are currently supported: @@ -462,6 +459,18 @@ supported: A list of command line options is provided by `docker/build.sh --help`. The container will install, configure and start all required dependencies, such as Hadoop. +By default, this script will run every module in the project. However, if you are planning on working on just a +small set of the modules (e.g. the GLVs) then you can use the script options to reduce the modules included by +specifically selecting which modules you want. This behavior is currently supported for the non-Java GLVs and +gremlin-console. This option will include only the selected modules as well as gremlin-server, gremlin-test, +neo4j-gremlin and all their dependencies. This is the minimum set of modules required to build and test the GLVs. + +[source,bash] +.Build and test gremlin-python and gremlin-go with minimal Gremlin Server dependencies +---- +./docker/build.sh --tests --integration-tests --python --golang +---- + Options can be passed to Docker by setting the `TINKERPOP_DOCKER_OPTS` environment variable. A speed boost can be gained at the expense of memory by using tmpfs and the special directory `/usr/src/tinkermem`. diff --git a/gremlin-go/run.sh b/gremlin-go/run.sh index e37b768711..5f62072dda 100755 --- a/gremlin-go/run.sh +++ b/gremlin-go/run.sh @@ -40,7 +40,7 @@ if [ "$1" == "-h" ] || [ "$1" == "--help" ]; then fi # Parses current gremlin server version from project pom.xml file using perl regex -GREMLIN_SERVER_VERSION=$(grep tinkerpop -A2 pom.xml | grep -Po '(?<=<version>)([0-9]+\.?){3}(-SNAPSHOT)?(?=<)') +GREMLIN_SERVER_VERSION=$(grep tinkerpop -A2 pom.xml | sed -r -n 's/.*<version>(([0-9]+\.?){3})(-SNAPSHOT)?<\/version>/\1\3/p') export GREMLIN_SERVER="${1:-$GREMLIN_SERVER_VERSION}" echo "Running server version: $GREMLIN_SERVER"