This is an automated email from the ASF dual-hosted git repository. pgj pushed a commit to branch build-with-clouseau in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 0bcb4792c4ef7450022d6c4b7070c0dc02e2f346 Author: Gabor Pali <gabor.p...@ibm.com> AuthorDate: Fri Nov 17 18:04:18 2023 +0100 Add Clouseau to the developer setup Provide a way to the developers to deploy and launch the Clouseau search module more easily, therefore making it more accessible. This can help with running the Search-based Elixir tests as well as the Mango `text` search tests. This could allow us to catch more bugs ahead of time and might even inspire more improvements in the area. The extension is designed in a way to make it simple to integrate with the CI — only the Java environment of the proper version needs to be deployed, everything else could be managed from this repository. Such as bumping the version of Clouseau or fine-tuning the configuration parameters. This is an opt-in feature on two levels. First, one has to tell `./configure --enable-clouseau` to instantiate Clouseau locally. This will create the `clouseau` sub-directory that holds the contents of the Clouseau distribution of the specified version, which currently defaults to 2.22.0 (the latest). If an older version is needed, use the `--clouseau-version` flag. Then the `mango-test` and `elixir-search` targets will try to use Clouseau automatically just to make it easy to use in the old way. However, the `dev/run` script will not do the same. It requires the `--with-clouseau` flag to be passed for that. That is because the developer may not necessarily want to launch Clouseau ad hoc, even if it is available. With this, note that `elixir-search` is added to the `check` target. In lack of a configured Clouseau instance, it will become a no-op and a warning is emitted. --- .devcontainer/devcontainer.json | 10 +-- Makefile | 25 ++++-- Makefile.win | 28 ++++-- README-DEV.rst | 68 ++++++++++++--- build-aux/Jenkinsfile.full | 18 ++++ build-aux/Jenkinsfile.pr | 4 +- configure | 75 ++++++++++++++++ configure.ps1 | 64 ++++++++++++++ dev/run | 184 +++++++++++++++++++++++++++++++++++++++- 9 files changed, 439 insertions(+), 37 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 491b2ecb6..4b947ce0c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,11 +5,11 @@ "context": "..", "args": { // Useful choices include: - // apache/couchdbci-debian:bullseye-erlang-25.0.2 - // apache/couchdbci-debian:bullseye-erlang-24.3.4.2 - // apache/couchdbci-debian:bullseye-erlang-23.3.4.15 + // apache/couchdbci-debian:bullseye-erlang-26.0 + // apache/couchdbci-debian:bullseye-erlang-25.3 + // apache/couchdbci-debian:bookworm-erlang-24.3.4.14 // - "COUCHDB_IMAGE": "apache/couchdbci-debian:bullseye-erlang-24.3.4.10" + "COUCHDB_IMAGE": "apache/couchdbci-debian:bullseye-erlang-24.3.4.14" } }, @@ -23,7 +23,7 @@ // Your code will live inside the volume created for the container under /workspace. "workspaceMount": "target=/workspaces/couchdb,type=volume", "workspaceFolder": "/workspaces/couchdb", - "postCreateCommand": "./configure --enable-nouveau && make", + "postCreateCommand": "./configure --enable-nouveau --enable-clouseau && make", "customizations": { "vscode": { diff --git a/Makefile b/Makefile index 83cc1d360..359cd19d6 100644 --- a/Makefile +++ b/Makefile @@ -157,6 +157,7 @@ check: all @$(MAKE) eunit @$(MAKE) mango-test @$(MAKE) elixir + @$(MAKE) elixir-search @$(MAKE) weatherreport-test @$(MAKE) nouveau-test @@ -263,12 +264,17 @@ elixir: elixir-init devclean --no-eval 'mix test --trace --include test/elixir/test/config/suite.elixir --exclude test/elixir/test/config/skip.elixir $(EXUNIT_OPTS)' .PHONY: elixir-search -# target: elixir-search - Run search tests, requires a running Clouseau instance +# target: elixir-search - Run search tests, requires a configured Clouseau instance elixir-search: export MIX_ENV=integration elixir-search: elixir-init devclean +ifeq ($(with_clouseau), 1) @dev/run -n 1 -q -a adm:pass \ + --with-clouseau \ --locald-config test/config/test-config.ini \ --no-eval 'mix test --trace --include test/elixir/test/config/search.elixir' +else + @echo "Warning: Clouseau is not enabled, \`elixir-search\` cannot be run." +endif .PHONY: elixir-source-checks # target: elixir-source-checks - Check source code formatting of Elixir test files @@ -313,19 +319,24 @@ list-eunit-suites: build-test: @test/build/test-configure.sh +ifeq ($(with_clouseau), 1) +_WITH_CLOUSEAU="--with-clouseau" +endif .PHONY: mango-test # target: mango-test - Run Mango tests mango-test: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1 mango-test: devclean all - @cd src/mango && \ - python3 -m venv .venv && \ - .venv/bin/python3 -m pip install -r requirements.txt - @cd src/mango && \ - ../../dev/run "$(TEST_OPTS)" \ + @python3 -m venv src/mango/.venv && \ + src/mango/.venv/bin/python3 -m pip install -r src/mango/requirements.txt + @dev/run \ + "$(TEST_OPTS)" \ + "$(_WITH_CLOUSEAU)" \ -n 1 \ --admin=adm:pass \ - 'COUCH_USER=adm COUCH_PASS=pass .venv/bin/python3 -m nose2 $(MANGO_TEST_OPTS)' + --no-eval "\ +COUCH_USER=adm COUCH_PASS=pass \ +src/mango/.venv/bin/nose2 -s src/mango/test -c src/mango/unittest.cfg $(MANGO_TEST_OPTS)" .PHONY: weatherreport-test diff --git a/Makefile.win b/Makefile.win index b0c6253a8..8f214e325 100644 --- a/Makefile.win +++ b/Makefile.win @@ -143,6 +143,7 @@ check: all @$(MAKE) eunit @$(MAKE) mango-test @$(MAKE) elixir + @$(MAKE) elixir-search @$(MAKE) nouveau-test ifdef apps @@ -203,6 +204,8 @@ python-black-update: .venv/bin/black --exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/erlfmt|src/rebar/pr2relnotes.py|src/fauxton" \ build-aux dev\run dev\format_*.py src\mango\test src\docs\src\conf.py src\docs\ext . +-include install.mk + ifeq ($(with_nouveau), 0) exclude_nouveau=--exclude nouveau endif @@ -239,12 +242,17 @@ elixir: elixir-init devclean --no-eval 'mix test --trace --include test\elixir\test\config\suite.elixir --exclude test\elixir\test\config\skip.elixir $(EXUNIT_OPTS)' .PHONY: elixir-search -# target: elixir-search - Run search tests, requires a running Clouseau instance +# target: elixir-search - Run search tests, requires a configured Clouseau instance elixir-search: export MIX_ENV=integration elixir-search: elixir-init devclean +ifeq ($(with_clouseau), 1) @dev\run -n 1 -q -a adm:pass \ + --with-clouseau \ --locald-config test/config/test-config.ini \ --no-eval 'mix test --trace --include test/elixir/test/config/search.elixir' +else + @echo "Warning: Clouseau is not enabled, `elixir-search` cannot be run." +endif .PHONY: elixir-source-checks # target: elixir-source-checks - Check source code formatting of Elixir test files @@ -278,19 +286,24 @@ list-eunit-apps: list-eunit-suites: @powershell -Command 'Get-ChildItem -Path src -Recurse -Filter "*_tests?.erl" | ForEach-Object { "{0}" -f $$_.BaseName } | Sort' +ifeq ($(with_clouseau), 1) +_WITH_CLOUSEAU="--with-clouseau" +endif .PHONY: mango-test # target: mango-test - Run Mango tests mango-test: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1 mango-test: devclean all - @cd src\mango && \ - python.exe -m venv .venv && \ - .venv\Scripts\pip.exe install -r requirements.txt - @cd src\mango && \ - ..\..\dev\run "$(TEST_OPTS)" \ + @$(PYTHON) -m venv src\mango\.venv && \ + src\mango\.venv\Scripts\pip.exe install -r src\mango\requirements.txt + @dev\run \ + "$(TEST_OPTS)" \ + "$(_WITH_CLOUSEAU)" \ -n 1 \ --admin=adm:pass \ - "env COUCH_USER=adm COUCH_PASS=pass .venv\Scripts\nose2 $(MANGO_TEST_OPTS)" + "\ +env COUCH_USER=adm COUCH_PASS=pass \ +src\mango\.venv\Scripts\nose2 -s src\mango\test -c src\mango\unittest.cfg $(MANGO_TEST_OPTS)" ################################################################################ @@ -350,7 +363,6 @@ dist: all derived .PHONY: release # target: release - Create an Erlang release including CouchDB! --include install.mk release: all @echo 'Installing CouchDB into rel\couchdb\ ...' -@rmdir /s/q rel\couchdb >NUL 2>&1 || true diff --git a/README-DEV.rst b/README-DEV.rst index d698173c1..3ab5c6bf6 100644 --- a/README-DEV.rst +++ b/README-DEV.rst @@ -152,9 +152,9 @@ Configure the source by running:: ./configure -If you intend to run the test suites:: +If you intend to run the test suites with Clouseau:: - ./configure -c + ./configure --enable-clouseau If you don't want to build Fauxton or documentation specify ``--disable-fauxton`` and/or ``--disable-docs`` arguments for ``configure`` to @@ -237,14 +237,8 @@ but it could be done manually via the corresponding target:: make elixir-search -Note that this requires a running Clouseau instance with the name -``clouseau@127.0.0.1``. The easiest way to get it is to clone the -`cloudant-labs/clouseau <https://github.com/cloudant-labs/clouseau>`_ -repository and launch it run there once all the prerequisites (JDK, -Scala, and Maven) have been installed successfully, e.g.:: - - git clone https://github.com/cloudant-labs/clouseau - mvn -f clouseau/pom.xml scala:run +Note that this requires Clouseau to be configured for running, see +above. Mango Integration Tests ~~~~~~~~~~~~~~~~~~~~~~~ @@ -262,7 +256,7 @@ the implementation. Consult its documentation for more information. Tests that rely on text indexes are run only if the ``search`` feature is reported to be available (i.e. a working Clouseau instance is -connected), otherwise they will be skipped. +configured and working), otherwise they will be skipped. Note that the databases that are created during the tests will be all removed after each of the suites completed. However, with the help of @@ -272,6 +266,58 @@ to keep those databases around for further investigation:: MANGO_TESTS_KEEP_DBS=please \ make mango-test MANGO_TEST_OPTS='03-operator-test' +Running Clouseau +~~~~~~~~~~~~~~~~ + +When configured with the ``./configure`` script, the ``./dev/run`` +script is capable of launching Clouseau instances alongside the +CouchDB nodes and hooking them up. This is what the ``mango-test`` +and ``elixir-search`` targets also use to run their respective test +suites, and let Clouseau automatically manage them. + +Although the ``./configure`` and the ``./dev/run`` scripts try to take +care of the details of the Clouseau deployment, it is still the +responsibility of the user to provide a suitable Java environment for +running. Clouseau can run with JRE 1.7 and 1.8 only. Also, when +Nouveau is in use, which uses a more recent Java environment, the old +JDK has to be installed separately and the ``CLOUSEAU_JAVA_HOME`` +environment variable has to be set to point its location. + +Fortunately, the ```asdf`` tool <https://asdf-vm.com/>` provides a +convenient way to install old versions of JDK through its ```java`` +plugin <https://github.com/halcyon/asdf-java>`:: + + asdf plugin add java + +Then use ``asdf`` to install it:: + + asdf install java zulu-jre-8.74.0.17 + +Finally, use ``asdf`` to set the ``CLOUSEAU_JAVA_HOME`` environment +variable:: + + export CLOUSEAU_JAVA_HOME=$(asdf where java zulu-jre-8.74.0.17) + +If the use of ``asdf`` is not an option, `the Zulu site +<https://cdn.azul.com/zulu/bin/>` could be used directly to get the +distribution package for the appropriate JRE version. But this is +just one of the possibilities to access installers for old Java +environments. + +Once both Clouseau and the corresponding Java environment are set, +they are not put in use automatically. In order to do so, the +``./dev/run`` script needs to be run with Clouseau enabled as +follows:: + + dev/run --with-clouseau + +When a specific Erlang cookie string is set in +``rel/overlay/etc/vm.args``, the ``--erlang-cookie`` flag could be +used to configure Clouseau to work with that:: + + dev/run --with-clouseau --erlang-cookie=brumbrum + + Static Code Analysis ~~~~~~~~~~~~~~~~~~~~ diff --git a/build-aux/Jenkinsfile.full b/build-aux/Jenkinsfile.full index 80b96a47d..5e1b213e2 100644 --- a/build-aux/Jenkinsfile.full +++ b/build-aux/Jenkinsfile.full @@ -27,6 +27,7 @@ meta = [ name: 'CentOS 7', spidermonkey_vsn: '1.8.5', enable_nouveau: true, + enable_clouseau: true, image: "apache/couchdbci-centos:7-erlang-${ERLANG_VERSION}" ], @@ -34,6 +35,7 @@ meta = [ name: 'CentOS 8', spidermonkey_vsn: '60', enable_nouveau: true, + enable_clouseau: true, image: "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}" ], @@ -41,6 +43,7 @@ meta = [ name: 'Ubuntu 18.04', spidermonkey_vsn: '1.8.5', enable_nouveau: true, + enable_clouseau: true, image: "apache/couchdbci-ubuntu:bionic-erlang-${ERLANG_VERSION}" ], @@ -48,6 +51,7 @@ meta = [ name: 'Ubuntu 20.04', spidermonkey_vsn: '68', enable_nouveau: true, + enable_clouseau: true, image: "apache/couchdbci-ubuntu:focal-erlang-${ERLANG_VERSION}" ], @@ -55,6 +59,7 @@ meta = [ name: 'Ubuntu 22.04', spidermonkey_vsn: '91', enable_nouveau: true, + enable_clouseau: true, image: "apache/couchdbci-ubuntu:jammy-erlang-${ERLANG_VERSION}" ], @@ -62,6 +67,7 @@ meta = [ name: 'Debian 10', spidermonkey_vsn: '60', enable_nouveau: true, + enable_clouseau: true, image: "apache/couchdbci-debian:buster-erlang-${ERLANG_VERSION}" ], @@ -72,6 +78,7 @@ meta = [ // name: 'Debian 11 ARM', // spidermonkey_vsn: '78', // enable_nouveau: true, + // enable_clouseau: true, // image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}", // node_label: 'arm64v8' // ], @@ -80,6 +87,7 @@ meta = [ name: 'Debian 11 POWER', spidermonkey_vsn: '78', enable_nouveau: true, + enable_clouseau: true, image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}", node_label: 'ppc64le' ], @@ -98,6 +106,7 @@ meta = [ name: 'Debian 11', spidermonkey_vsn: '78', enable_nouveau: true, + enable_clouseau: true, image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}" ], @@ -105,18 +114,21 @@ meta = [ name: 'Debian 12', spidermonkey_vsn: '78', enable_nouveau: true, + enable_clouseau: true, image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}" ], 'freebsd-x86_64': [ name: 'FreeBSD', spidermonkey_vsn: '91', + enable_clouseau: false, gnu_make: 'gmake' ], 'freebsd-arm64': [ name: 'FreeBSD', spidermonkey_vsn: '91', + enable_clouseau: false, gnu_make: 'gmake' ], @@ -124,6 +136,7 @@ meta = [ name: 'macOS', spidermonkey_vsn: '91', enable_nouveau: false, + enable_clouseau: false, gnu_make: 'make' ] ] @@ -133,6 +146,9 @@ def String configure(config) { if (config.enable_nouveau) { result += " --enable-nouveau" } + if (config.enable_clouseau) { + result += " --enable-clouseau" + } return result } @@ -170,6 +186,7 @@ def generateNativeStage(platform) { sh '$MAKE' sh '$MAKE eunit' sh '$MAKE elixir' + sh '$MAKE elixir-search' sh '$MAKE mango-test' sh '$MAKE weatherreport-test' sh '$MAKE nouveau-test' @@ -223,6 +240,7 @@ def generateContainerStage(platform) { sh 'make' sh 'make eunit' sh 'make elixir' + sh 'make elixir-search' sh 'make mango-test' sh 'make weatherreport-test' sh 'make nouveau-test' diff --git a/build-aux/Jenkinsfile.pr b/build-aux/Jenkinsfile.pr index d1313d480..577d4aad4 100644 --- a/build-aux/Jenkinsfile.pr +++ b/build-aux/Jenkinsfile.pr @@ -20,7 +20,7 @@ mkdir build cd build tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz cd apache-couchdb-* -./configure --enable-nouveau +./configure --enable-nouveau --enable-clouseau make check || (make build-report && false) ''' @@ -218,7 +218,7 @@ pipeline { steps { sh ''' rm -rf apache-couchdb-* - ./configure --spidermonkey-version 78 --enable-nouveau + ./configure --spidermonkey-version 78 --enable-nouveau --enable-clouseau make dist chmod -R a+w * . ''' diff --git a/configure b/configure index 8ed9817d2..b4fa5c629 100755 --- a/configure +++ b/configure @@ -29,6 +29,7 @@ WITH_PROPER="true" WITH_FAUXTON=1 WITH_DOCS=1 WITH_NOUVEAU=0 +WITH_CLOUSEAU=0 ERLANG_MD5="false" SKIP_DEPS=0 @@ -38,6 +39,8 @@ run_erlang() { COUCHDB_USER="$(whoami 2>/dev/null || echo couchdb)" SM_VSN=${SM_VSN:-"91"} +CLOUSEAU_VSN=${CLOUSEAU_VSN:-"2.22.0"} +CLOUSEAU_DIR="$(pwd)"/clouseau ARCH="$(uname -m)" ERLANG_VER="$(run_erlang 'io:put_chars(erlang:system_info(otp_release)).')" ERLANG_OS="$(run_erlang 'case os:type() of {OS, _} -> io:format("~s~n", [OS]) end.')" @@ -59,9 +62,11 @@ Options: --disable-fauxton do not build Fauxton --disable-docs do not build any documentation or manpages --enable-nouveau enable the new experimental search module + --enable-clouseau enable the Clouseau search module --erlang-md5 use erlang for md5 hash operations --dev alias for --disable-docs --disable-fauxton --spidermonkey-version VSN specify the version of SpiderMonkey to use (defaults to $SM_VSN) + --clouseau-version VSN specify the version of Clouseau to use (defaults to $CLOUSEAU_VSN) --skip-deps do not update erlang dependencies --rebar=PATH use rebar by specified path (version >=2.6.0 && <3.0 required) --rebar3=PATH use rebar3 by specified path @@ -101,6 +106,12 @@ parse_opts() { continue ;; + --enable-clouseau) + WITH_CLOUSEAU=1 + shift + continue + ;; + --erlang-md5) ERLANG_MD5="true" shift @@ -202,6 +213,24 @@ parse_opts() { exit 1 ;; + --clouseau-version) + if [ -n "$2" ]; then + eval CLOUSEAU_SVN=$2 + shift 2 + continue + else + printf 'ERROR: "--clouseau-version" requires a non-empty argument.\n' >&2 + exit 1 + fi + ;; + --clouseau-version=?*) + eval CLOUSEAU_VSN=${1#*=} + ;; + --clouseau-version=) + printf 'ERROR: "--clouseau-version" requires a non-empty argument.\n' >&2 + exit 1 + ;; + --) # End of options shift break @@ -298,6 +327,7 @@ package_author_name = $PACKAGE_AUTHOR_NAME with_fauxton = $WITH_FAUXTON with_docs = $WITH_DOCS with_nouveau = $WITH_NOUVEAU +with_clouseau = $WITH_CLOUSEAU user = $COUCHDB_USER spidermonkey_version = $SM_VSN @@ -345,6 +375,45 @@ install_local_erlfmt() { fi } +install_local_clouseau() { + _DIST_URL=https://github.com/cloudant-labs/clouseau/releases/download/"$CLOUSEAU_VSN"/clouseau-"$CLOUSEAU_VSN"-dist.zip + _MAVEN_BASE_URI=https://repo1.maven.org/maven2 + + _SLF4J_SIMPLE_VSN=${SLF4J_SIMPLE_VERSION:-1.7.36} + _SLF4J_SIMPLE_JAR=slf4j-simple-"$_SLF4J_SIMPLE_VSN".jar + _SLF4J_SIMPLE_URL="$_MAVEN_BASE_URI"/org/slf4j/slf4j-simple/"$_SLF4J_SIMPLE_VSN"/"$_SLF4J_SIMPLE_JAR" + + rm -rf "$CLOUSEAU_DIR" + mkdir -p "$CLOUSEAU_DIR" + + if ! curl -sSL --max-redirs 1 -o clouseau.zip "$_DIST_URL"; then + printf "ERROR: %s could not be downloaded.\n" "$_DIST_URL" >&2 + exit 1 + fi + + if ! unzip -q -j clouseau.zip -d "$CLOUSEAU_DIR"; then + printf "ERROR: Clouseau distribution package (clouseau.zip) could not be extracted.\n" >&2 + exit 1 + fi + + rm clouseau.zip + + if ! curl -sSL --max-redirs 1 -o "$CLOUSEAU_DIR"/"$_SLF4J_SIMPLE_JAR" "$_SLF4J_SIMPLE_URL"; then + printf "ERROR: %s could not be downloaded.\n" "$_SLF4J_SIMPLE_URL" >&2 + exit 1 + fi + + cat <<EOF > "$CLOUSEAU_DIR"/clouseau.ini +[clouseau] +EOF + cat <<EOF > "$CLOUSEAU_DIR"/log4j.properties +log4j.rootLogger=debug, CONSOLE +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} %c [%p] %m%n +EOF +} + if [ -z "${REBAR}" ]; then install_local_rebar REBAR=${rootdir}/bin/rebar @@ -360,6 +429,12 @@ if [ -z "${ERLFMT}" ]; then ERLFMT=${rootdir}/bin/erlfmt fi +if [ $WITH_CLOUSEAU -ne 0 ]; then + install_local_clouseau +else + rm -rf "$CLOUSEAU_DIR" +fi + # only update dependencies, when we are not in a release tarball if [ -d .git -a $SKIP_DEPS -ne 1 ]; then echo "==> updating dependencies" diff --git a/configure.ps1 b/configure.ps1 index ec0b650db..7d13b5505 100644 --- a/configure.ps1 +++ b/configure.ps1 @@ -8,9 +8,11 @@ -DisableFauxton request build process skip building Fauxton (default false) -DisableDocs request build process skip building documentation (default false) -EnableNouveau enable the new experiemtal search module (default false) + -EnableClouseau enable the Clouseau search module (default false) -SkipDeps do not update Erlang dependencies (default false) -CouchDBUser USER set the username to run as (defaults to current user) -SpiderMonkeyVersion VSN select the version of SpiderMonkey to use (default 91) + -ClouseauVersion VSN select the version of Clouseau to use (default 2.22.0) Installation directories: -Prefix PREFIX install architecture-independent files in PREFIX @@ -45,6 +47,7 @@ Param( [switch]$DisableFauxton = $false, # do not build Fauxton [switch]$DisableDocs = $false, # do not build any documentation or manpages [switch]$EnableNouveau = $false, # dont use new search module by default + [switch]$EnableClouseau = $false, # do not use Clouseau by default [switch]$SkipDeps = $false, # do not update erlang dependencies [switch]$DisableProper = $false, # a compilation pragma. proper is a kind of automated test suite [switch]$EnableErlangMD5 = $false, # don't use Erlang for md5 hash operations by default @@ -54,6 +57,8 @@ Param( [ValidateNotNullOrEmpty()] [string]$SpiderMonkeyVersion = "91", # select the version of SpiderMonkey to use (default 91) [ValidateNotNullOrEmpty()] + [string]$ClouseauVersion = "2.22.0", # select the version of Clouseau to use (default 2.22.0) + [ValidateNotNullOrEmpty()] [string]$Prefix = "C:\Program Files\Apache\CouchDB", # install architecture-independent file location (default C:\Program Files\Apache\CouchDB) [ValidateNotNullOrEmpty()] [string]$ExecPrefix = $Prefix, # install architecture-dependent file location (default C:\Program Files\Apache\CouchDB) @@ -130,6 +135,7 @@ $LogFile="$LogDir\couch.log" $BuildFauxton = [int](-not $DisableFauxton) $BuildDocs = [int](-not $DisableDocs) $BuildNouveau = $(If ($EnableNouveau) {1} else {0}) +$WithClouseau = $(If ($EnableClouseau) {1} else {0}) $Hostname = [System.Net.Dns]::GetHostEntry([string]"localhost").HostName $WithProper = (-not $DisableProper).ToString().ToLower() $ErlangMD5 = ($EnableErlangMD5).ToString().ToLower() @@ -201,6 +207,7 @@ man_dir = $ManDir with_fauxton = $BuildFauxton with_docs = $BuildDocs with_nouveau = $BuildNouveau +with_clouseau = $WithClouseau user = $CouchDBUser spidermonkey_version = $SpiderMonkeyVersion @@ -267,6 +274,63 @@ if ((Get-Command "erlfmt.cmd" -ErrorAction SilentlyContinue) -eq $null) cd ..\.. } +$ClouseauDir = "$rootdir\clouseau" + +if ($EnableClouseau) +{ + Write-Verbose "===> downloading Clouseau distribution..." + + if (Test-Path $ClouseauDir) { + Remove-Item -Recurse -Force $ClouseauDir + } + New-Item -Path $ClouseauDir -ItemType Directory | Out-File Null + + $Slf4jVersion = "1.7.36" + $ClouseauDistUrl = "https://github.com/cloudant-labs/clouseau/releases/download/$ClouseauVersion/clouseau-$ClouseauVersion-dist.zip" + $Slf4jSimpleJar = "slf4j-simple-$Slf4jVersion.jar" + $Slf4jSimpleUrl = "https://repo1.maven.org/maven2/org/slf4j/slf4j-simple/$Slf4jVersion/$Slf4jSimpleJar" + + Set-Variable ProgressPreference SilentlyContinue + Invoke-WebRequest -MaximumRedirection 1 -OutFile clouseau.zip $ClouseauDistUrl + If ($LASTEXITCODE -ne 0) { + Write-Output "ERROR: $ClouseauDistUrl could not be downloaded." + exit 1 + } + + Expand-Archive clouseau.zip -DestinationPath $ClouseauDir -Force + If ($LASTEXITCODE -ne 0) { + Write-Output "ERROR: Clouseau distribution package (clouseau.zip) could not be extracted." + exit 1 + } + mv "$ClouseauDir\*\*.jar" "$ClouseauDir" + rm "$ClouseauDir\clouseau-$ClouseauVersion" + rm clouseau.zip + + Invoke-WebRequest -MaximumRedirection 1 -OutFile "$ClouseauDir\$Slf4jSimpleJar" $Slf4jSimpleUrl + If ($LASTEXITCODE -ne 0) { + Write-Output "ERROR: $Slf4jSimpleJarUrl could not be downloaded." + exit 1 + } + + $ClouseauIni = @" +[clouseau] +"@ + $ClouseauIni | Out-File "$ClouseauDir\clouseau.ini" -encoding ascii + + $Log4JProperties = @" +log4j.rootLogger=debug, CONSOLE +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} %c [%p] %m%n +"@ + $Log4JProperties | Out-File "$ClouseauDir\log4j.properties" +} +else { + if (Test-Path $ClouseauDir) { + Remove-Item -Recurse -Force $ClouseauDir + } +} + # only update dependencies, when we are not in a release tarball if ( (Test-Path .git -PathType Container) -and (-not $SkipDeps) ) { Write-Verbose "==> updating dependencies" diff --git a/dev/run b/dev/run index 707c8a4d7..7bb807bf2 100755 --- a/dev/run +++ b/dev/run @@ -23,6 +23,7 @@ import json import ntpath import optparse import os +import platform import posixpath import re import signal @@ -90,11 +91,15 @@ log.verbose = True def main(): ctx = setup() - startup(ctx) - if ctx["cmd"]: - run_command(ctx, ctx["cmd"]) + try: + startup(ctx) + except ClouseauError: + sys.exit(1) else: - join(ctx, cluster_port(ctx, 1), *ctx["admin"]) + if ctx["cmd"]: + run_command(ctx, ctx["cmd"]) + else: + join(ctx, cluster_port(ctx, 1), *ctx["admin"]) def setup(): @@ -235,6 +240,19 @@ def get_args_parser(): action="store_true", help="Start Nouveau server", ) + parser.add_option( + "--with-clouseau", + dest="with_clouseau", + default=False, + action="store_true", + help="Start Clouseau nodes", + ) + parser.add_option( + "--erlang-cookie", + dest="erlang_cookie", + default=None, + help="Erlang cookie string", + ) parser.add_option( "-t", "--enable-tls", @@ -279,6 +297,8 @@ def setup_context(opts, args): "auto_ports": opts.auto_ports, "locald_configs": opts.locald_configs, "with_nouveau": opts.with_nouveau, + "with_clouseau": opts.with_clouseau, + "erlang_cookie": opts.erlang_cookie, "enable_tls": opts.enable_tls, "no_tls": opts.no_tls, } @@ -505,6 +525,157 @@ def boot_nouveau(ctx): ) +CLOUSEAU_DIR = "clouseau" +JAVA_VERSION_RE = re.compile('"(\d+\.\d+).*"') + + +def get_java_version(java): + try: + output = sp.check_output([java, "-version"], stderr=sp.STDOUT).decode("utf-8") + except: + output = None + + if output: + matches = JAVA_VERSION_RE.search(output) + if matches: + return float(matches.groups()[0]) + + +class ClouseauError(Exception): + def __init__(self, message): + self.message = message + super().__init__(self.message) + + +@log("Start Clouseau node clouseau{idx}") +def boot_clouseau(ctx, idx): + configure_cmd = ( + "&.\\configure.ps1 -EnableClouseau" + if platform.system() == "Windows" + else "./configure --enable-clouseau" + ) + + if not os.path.isdir(CLOUSEAU_DIR): + raise ClouseauError( + "Clouseau deployment cannot be found, please run `{}`".format(configure_cmd) + ) + + clouseau_jdk_home = os.environ.get("CLOUSEAU_JAVA_HOME") or os.environ.get( + "JAVA_HOME" + ) + java = clouseau_jdk_home + "/bin/java" if clouseau_jdk_home else "java" + java_version = get_java_version(java) + + if not java_version: + print( + "Warning: Java version could not be determined, Clouseau may not be able to run" + ) + else: + if java_version < 1.7 or java_version > 1.8: + raise ClouseauError( + "Java is not suitable to run Clouseau. Please use JRE 1.7 or 1.8 and configure its (root) path in `CLOUSEAU_JAVA_HOME`" + ) + + clouseau_jars = [ + "{}/{}".format(CLOUSEAU_DIR, fname) + for fname in os.listdir(CLOUSEAU_DIR) + if fname.endswith(".jar") + ] + + if not clouseau_jars: + raise ClouseauError("Clouseau has no JAR files") + + clouseau_ini = "{}/clouseau.ini".format(CLOUSEAU_DIR) + if not os.path.isfile(clouseau_ini): + raise ClouseauError("Clouseau has no ini file") + + log4j_properties = "{}/log4j.properties".format(CLOUSEAU_DIR) + if not os.path.isfile(log4j_properties): + raise ClouseauError("Clouseau has no Log4J configuration") + + clouseau_name = "clouseau{}@127.0.0.1".format(idx) + clouseau_indexes_dir = "{}/clouseau{}/data".format(ctx["devdir"], idx) + + if ctx["erlang_cookie"]: + clouseau_cookie = ["-Dclouseau.cookie={}".format(ctx["erlang_cookie"])] + else: + clouseau_cookie = [] + + # `HOME` must be set for Scalang to find the Erlang cookie + if not os.environ.get("HOME"): + # This is usually `USERPROFILE` on Windows + alternativeHome = os.environ.get("USERPROFILE") + if alternativeHome: + os.environ["HOME"] = alternativeHome + + separator = ";" if platform.system() == "Windows" else ":" + + cmd = ( + [ + java, + "-server", + "-Xmx2G", + "-Dsun.net.inetaddr.ttl=30", + "-Dsun.net.inetaddr.negative.ttl=30", + '-XX:OnOutOfMemoryError="kill -9 %p"', + "-XX:+UseConcMarkSweepGC", + "-XX:+CMSParallelRemarkEnabled", + "-cp", + separator.join(clouseau_jars), + "-Dlog4j.configuration=file:{}".format(log4j_properties), + "-Dclouseau.name={}".format(clouseau_name), + "-Dclouseau.dir={}".format(clouseau_indexes_dir), + ] + + clouseau_cookie + + [ + "com.cloudant.clouseau.Main", + clouseau_ini, + ] + ) + logfname = os.path.join(ctx["devdir"], "logs", "clouseau{}.log".format(idx)) + log = open(logfname, "w") + + try: + return sp.Popen( + cmd, + stdin=sp.PIPE, + stdout=log, + stderr=sp.STDOUT, + ) + except: + raise ClouseauError("Java is not capable of running Clouseau") + + +def maybe_boot_clouseau(ctx, idx): + if ctx["with_clouseau"]: + return boot_clouseau(ctx, idx) + + +@log("Check Clouseau node clouseau{idx}") +def check_clouseau_node_alive(ctx, idx): + if ctx["erlang_cookie"]: + cookie = ["-c", ctx["erlang_cookie"]] + else: + cookie = [] + + cmd = ( + ["erl_call", "-n", "node{}@127.0.0.1".format(idx)] + + cookie + + [ + "-a", + "net_adm ping ['clouseau{}@127.0.0.1']".format(idx), + ] + ) + + if sp.check_output(cmd) != b"pong": + raise ValueError("Clouseau is not working") + + +def maybe_check_clouseau_node_alive(ctx, idx): + if ctx["with_clouseau"]: + check_clouseau_node_alive(ctx, idx) + + def hack_default_ini(ctx, node, contents): contents = re.sub( "^\[httpd\]$", @@ -628,6 +799,10 @@ def boot_nodes(ctx): nouveau_proc = boot_nouveau(ctx) if nouveau_proc is not None: ctx["procs"].append(nouveau_proc) + for idx in [(i + ctx["node_number"]) for i in range(ctx["N"])]: + clouseau_proc = maybe_boot_clouseau(ctx, idx) + if clouseau_proc is not None: + ctx["procs"].append(clouseau_proc) def ensure_all_nodes_alive(ctx): @@ -640,6 +815,7 @@ def ensure_all_nodes_alive(ctx): url = "http://127.0.0.1:{0}/".format(local_port) try: check_node_alive(url) + maybe_check_clouseau_node_alive(ctx, num + 1) except: pass else: