This is an automated email from the ASF dual-hosted git repository.
joaoreis pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-gocql-driver.git
commit e517c94082d1889a649f4ff4a6df858db2fbb9ef
Author: Dmitry Kropachev <dmitry.kropac...@gmail.com>
AuthorDate: Wed May 28 13:17:39 2025 -0400
Move all cicd steps into Makefile
1. Implement all steps in Makefile
2. Make CICD use Makefile
3. Drop setup-environment step, it is done automatically now
4. Add linting step
---
.github/actions/setup-environment/action.yml | 50 -------
.github/workflows/main.yml | 188 +++++++++------------------
Makefile | 151 +++++++++++++++++++++
3 files changed, 213 insertions(+), 176 deletions(-)
diff --git a/.github/actions/setup-environment/action.yml
b/.github/actions/setup-environment/action.yml
deleted file mode 100644
index 3ed57aac..00000000
--- a/.github/actions/setup-environment/action.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-name: Setup environment
-description: Setup environment for integration tests execution
-runs:
- using: "composite"
- steps:
- - name: Set up cache for SDKMAN
- uses: actions/cache@v3
- with:
- path: ~/.sdkman
- key: ${{ runner.os }}-sdkman
-
- - name: Set up cache for PIP
- uses: actions/cache@v3
- with:
- path: ~/.cache/pip
- key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
- restore-keys: |
- ${{ runner.os }}-pip-
-
- - name: Install Java
- shell: bash
- run: |
- echo "Installing SDKMAN..."
- curl -s "https://get.sdkman.io"; | bash
- source "$HOME/.sdkman/bin/sdkman-init.sh"
- echo "sdkman_auto_answer=true" >> ~/.sdkman/etc/config
-
- echo "Installing Java versions..."
- sdk install java 11.0.24-zulu
- sdk install java 17.0.12-zulu
-
- sdk default java 11.0.24-zulu
- sdk use java 11.0.24-zulu
-
- echo "JAVA11_HOME=$JAVA_HOME_11_X64" >> $GITHUB_ENV
- echo "JAVA17_HOME=$JAVA_HOME_17_X64" >> $GITHUB_ENV
- echo "JAVA_HOME=$JAVA_HOME_11_X64" >> $GITHUB_ENV
- echo "PATH=$PATH" >> $GITHUB_ENV
-
- - name: Install CCM
- shell: bash
- run: |
- echo "Creating Python virtual environment..."
- VENV_DIR="$HOME/venv"
- python3 -m venv $VENV_DIR
- source $VENV_DIR/bin/activate
- pip install --upgrade pip setuptools
-
- echo "Installing CCM..."
- pip install "git+https://github.com/riptano/ccm.git@${CCM_VERSION}";
\ No newline at end of file
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index db4228dc..d9ae88a8 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -7,9 +7,6 @@ on:
pull_request:
types: [ opened, synchronize, reopened ]
-env:
- CCM_VERSION: "4621dfee5ad73956b831091a8b863d100d25c610"
-
jobs:
build:
name: Unit tests
@@ -22,9 +19,11 @@ jobs:
- uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go }}
- - run: go vet ./...
+ - name: Run linting
+ run: make check
- name: Run unit tests
- run: go test -v -tags unit -race ./...
+ run: make test-unit
+
integration-cassandra:
timeout-minutes: 15
needs:
@@ -45,82 +44,51 @@ jobs:
compressor: "snappy"
steps:
- uses: actions/checkout@v2
- - uses: actions/setup-go@v2
+
+ - uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go }}
- - uses: actions/cache@v4
- id: gomod-cache
- with:
- path: ~/go/pkg/mod
- key: ${{ runner.os }}-go-${{ hashFiles('go.mod') }}
- restore-keys: |
- ${{ runner.os }}-go-
- - name: Setup environment
- uses: ./.github/actions/setup-environment
- - name: Start cassandra nodes
- run: |
- source ~/venv/bin/activate
- VERSION=${{ matrix.cassandra_version }}
- keypath="$(pwd)/testdata/pki"
- conf=(
- "client_encryption_options.enabled: true"
- "client_encryption_options.keystore: $keypath/.keystore"
- "client_encryption_options.keystore_password: cassandra"
- "client_encryption_options.require_client_auth: true"
- "client_encryption_options.truststore: $keypath/.truststore"
- "client_encryption_options.truststore_password: cassandra"
- "concurrent_reads: 2"
- "concurrent_writes: 2"
- "write_request_timeout_in_ms: 5000"
- "read_request_timeout_in_ms: 5000"
- )
-
- if [[ $VERSION == 3.*.* ]]; then
- conf+=(
- "rpc_server_type: sync"
- "rpc_min_threads: 2"
- "rpc_max_threads: 2"
- "enable_user_defined_functions: true"
- "enable_materialized_views: true"
- )
- elif [[ $VERSION == 4.0.* ]]; then
- conf+=(
- "enable_user_defined_functions: true"
- "enable_materialized_views: true"
- )
- else
- conf+=(
- "user_defined_functions_enabled: true"
- "materialized_views_enabled: true"
- )
- fi
- ccm remove test || true
-
- ccm create test -v $VERSION -n 3 -d --vnodes --jvm_arg="-Xmx256m
-XX:NewSize=100m"
- ccm updateconf "${conf[@]}"
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.10'
+ cache: 'pip'
+ cache-dependency-path: Makefile
- export JVM_EXTRA_OPTS=" -Dcassandra.test.fail_writes_ks=test
-Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"
+ - name: Set up cache for SDKMAN
+ uses: actions/cache@v3
+ with:
+ path: ~/.sdkman
+ key: ${{ runner.os }}-sdkman
- ccm start --wait-for-binary-proto --verbose
- ccm status
- ccm node1 nodetool status
+ - name: Set up cache for CCM
+ uses: actions/cache@v4
+ with:
+ path: ~/.ccm/repository
+ key: ${{ runner.os }}-ccm-${{ matrix.cassandra_version }}
- args="-gocql.timeout=60s -runssl -proto=${{ matrix.proto_version }}
-rf=3 -clusterSize=3 -autowait=2000ms -compressor=${{ matrix.compressor }}
-gocql.cversion=$VERSION -cluster=$(ccm liveset) ./..."
+ - name: Start cassandra nodes
+ env:
+ CASSANDRA_VERSION: ${{ matrix.cassandra_version }}
+ run: |
+ make cassandra-start
+ sleep 30s
- echo "args=$args" >> $GITHUB_ENV
- echo "JVM_EXTRA_OPTS=$JVM_EXTRA_OPTS" >> $GITHUB_ENV
- name: Integration tests
+ env:
+ CASSANDRA_VERSION: ${{ matrix.cassandra_version }}
+ TEST_CQL_PROTOCOL: ${{ matrix.proto_version }}
+ TEST_COMPRESSOR: ${{ matrix.compressor }}
+ TEST_INTEGRATION_TAGS: ${{ matrix.tags }}
run: |
- source ~/venv/bin/activate
- export JVM_EXTRA_OPTS="${{env.JVM_EXTRA_OPTS}}"
- go test -v -tags "${{ matrix.tags }} gocql_debug" -timeout=5m -race
${{ env.args }}
+ make test-integration
+
- name: 'Save ccm logs'
if: 'failure()'
uses: actions/upload-artifact@v4
with:
name: ccm-cluster-cassandra-${{ matrix.cassandra_version }}-go-${{
matrix.go }}-tag-${{ matrix.tags }}-proto-version-${{ matrix.proto_version
}}-compressor-${{ matrix.compressor }}
- path: /home/runner/.ccm/test
+ path: /home/runner/.ccm/gocql_integration_test
retention-days: 5
integration-auth-cassandra:
timeout-minutes: 15
@@ -144,69 +112,37 @@ jobs:
- uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go }}
- - name: Setup environment
- uses: ./.github/actions/setup-environment
- - name: Start cassandra nodes
- run: |
- source ~/venv/bin/activate
- VERSION=${{ matrix.cassandra_version }}
- keypath="$(pwd)/testdata/pki"
- conf=(
- "client_encryption_options.enabled: true"
- "client_encryption_options.keystore: $keypath/.keystore"
- "client_encryption_options.keystore_password: cassandra"
- "client_encryption_options.require_client_auth: true"
- "client_encryption_options.truststore: $keypath/.truststore"
- "client_encryption_options.truststore_password: cassandra"
- "concurrent_reads: 2"
- "concurrent_writes: 2"
- "write_request_timeout_in_ms: 5000"
- "read_request_timeout_in_ms: 5000"
- "authenticator: PasswordAuthenticator"
- "authorizer: CassandraAuthorizer"
- "enable_user_defined_functions: true"
- )
-
- if [[ $VERSION == 3.*.* ]]; then
- conf+=(
- "rpc_server_type: sync"
- "rpc_min_threads: 2"
- "rpc_max_threads: 2"
- "enable_user_defined_functions: true"
- "enable_materialized_views: true"
- )
- elif [[ $VERSION == 4.0.* ]]; then
- conf+=(
- "enable_user_defined_functions: true"
- "enable_materialized_views: true"
- )
- else
- conf+=(
- "user_defined_functions_enabled: true"
- "materialized_views_enabled: true"
- )
- fi
-
- ccm remove test || true
-
- ccm create test -v $VERSION -n 1 -d --vnodes --jvm_arg="-Xmx256m
-XX:NewSize=100m"
- ccm updateconf "${conf[@]}"
-
- rm -rf $HOME/.ccm/test/node1/data/system_auth
- export JVM_EXTRA_OPTS=" -Dcassandra.test.fail_writes_ks=test
-Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.10'
+ cache: 'pip'
+ cache-dependency-path: Makefile
- ccm start --wait-for-binary-proto --verbose
- ccm status
- ccm node1 nodetool status
+ - name: Set up cache for SDKMAN
+ uses: actions/cache@v3
+ with:
+ path: ~/.sdkman
+ key: ${{ runner.os }}-sdkman
- args="-gocql.timeout=60s -runssl -proto=${{ matrix.proto_version }}
-rf=3 -clusterSize=1 -autowait=2000ms -compressor=${{ matrix.compressor }}
-gocql.cversion=$VERSION -cluster=$(ccm liveset) ./..."
+ - name: Set up cache for CCM
+ uses: actions/cache@v4
+ with:
+ path: ~/.ccm/repository
+ key: ${{ runner.os }}-ccm-${{ matrix.cassandra_version }}
- echo "args=$args" >> $GITHUB_ENV
- echo "JVM_EXTRA_OPTS=$JVM_EXTRA_OPTS" >> $GITHUB_ENV
+ - name: Start cassandra nodes
+ env:
+ CASSANDRA_VERSION: ${{ matrix.cassandra_version }}
+ run: |
+ make cassandra-start
sleep 30s
+
- name: Integration tests
+ env:
+ CASSANDRA_VERSION: ${{ matrix.cassandra_version }}
+ TEST_CQL_PROTOCOL: ${{ matrix.proto_version }}
+ TEST_COMPRESSOR: ${{ matrix.compressor }}
+ TEST_INTEGRATION_TAGS: ${{ matrix.tags }}
run: |
- source ~/venv/bin/activate
- export JVM_EXTRA_OPTS="${{env.JVM_EXTRA_OPTS}}"
- go test -v -run=TestAuthentication -tags "${{ matrix.tags }}
gocql_debug" -timeout=15s -runauth ${{ env.args }}
+ make test-integration-auth
diff --git a/Makefile b/Makefile
new file mode 100644
index 00000000..4ef54d28
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,151 @@
+SHELL := bash
+MAKEFILE_PATH := $(abspath $(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
+KEY_PATH = ${MAKEFILE_PATH}/testdata/pki
+
+CASSANDRA_VERSION ?= 4.1.6
+TEST_CQL_PROTOCOL ?= 4
+TEST_COMPRESSOR ?= no-compression
+TEST_INTEGRATION_TAGS ?= integration
+
+CCM_VERSION ?= 39b8222b31a6c7afe8fe845d16981088a5a735ad
+JVM_EXTRA_OPTS ?= -Dcassandra.test.fail_writes_ks=test
-Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler
+ifeq (${CCM_CONFIG_DIR},)
+ CCM_CONFIG_DIR = ~/.ccm
+endif
+CCM_CONFIG_DIR := $(shell readlink --canonicalize ${CCM_CONFIG_DIR})
+
+CASSANDRA_CONFIG ?= "client_encryption_options.enabled: true" \
+"client_encryption_options.keystore: ${KEY_PATH}/.keystore" \
+"client_encryption_options.keystore_password: cassandra" \
+"client_encryption_options.require_client_auth: true" \
+"client_encryption_options.truststore: ${KEY_PATH}/.truststore" \
+"client_encryption_options.truststore_password: cassandra" \
+"concurrent_reads: 2" \
+"concurrent_writes: 2" \
+"write_request_timeout_in_ms: 5000" \
+"read_request_timeout_in_ms: 5000"
+
+ifeq ($(shell echo "${CASSANDRA_VERSION}" | grep -oP "3\.[0-9]+\.[0-9]+"
),${CASSANDRA_VERSION})
+ CASSANDRA_CONFIG += "rpc_server_type: sync" \
+"rpc_min_threads: 2" \
+"rpc_max_threads: 2" \
+"enable_user_defined_functions: true" \
+"enable_materialized_views: true"
+else ifeq ($(shell echo "${CASSANDRA_VERSION}" | grep -oP "4\.0\.[0-9]+"
),${CASSANDRA_VERSION})
+ CASSANDRA_CONFIG += "enable_user_defined_functions: true" \
+"enable_materialized_views: true"
+else
+ CASSANDRA_CONFIG += "user_defined_functions_enabled: true" \
+"materialized_views_enabled: true"
+endif
+
+ifneq (${JAVA11_HOME},)
+ export JAVA11_HOME
+endif
+
+ifneq (${JAVA17_HOME},)
+ export JAVA17_HOME
+endif
+
+ifneq (${JAVA_HOME},)
+ export JAVA_HOME
+endif
+
+export JVM_EXTRA_OPTS
+
+.prepare-cassandra-cluster: .prepare-ccm .prepare-java
+ @if [ -d ${CCM_CONFIG_DIR}/gocql_integration_test ] && ccm switch
gocql_integration_test 2>/dev/null 1>&2 && ccm status | grep UP 2>/dev/null
1>&2; then \
+ echo "Cassandra cluster is already started"; \
+ else \
+ echo "Start cassandra ${CASSANDRA_VERSION} cluster"; \
+ ccm stop gocql_integration_test 2>/dev/null 1>&2 || true; \
+ ccm remove gocql_integration_test 2>/dev/null 1>&2 || true; \
+ rm -rf ${CCM_CONFIG_DIR}/gocql_integration_test || true; \
+ ccm create gocql_integration_test -v ${CASSANDRA_VERSION} -n 3
-d --vnodes --jvm_arg="-Xmx256m -XX:NewSize=100m" && \
+ ccm updateconf ${CASSANDRA_CONFIG} && \
+ ccm start --wait-for-binary-proto --verbose && \
+ ccm status && \
+ ccm node1 nodetool status; \
+ fi
+
+cassandra-start: .prepare-ccm .prepare-java
+ @echo "Start cassandra ${CASSANDRA_VERSION} cluster"
+ @ccm stop gocql_integration_test 2>/dev/null 1>&2 || true
+ @ccm remove gocql_integration_test 2>/dev/null 1>&2 || true
+ @rm -rf ${CCM_CONFIG_DIR}/gocql_integration_test || true
+ ccm create gocql_integration_test -v ${CASSANDRA_VERSION} -n 3 -d
--vnodes --jvm_arg="-Xmx256m -XX:NewSize=100m"
+ @ccm updateconf ${CASSANDRA_CONFIG}
+ @ccm start --wait-for-binary-proto --verbose
+ @ccm status
+ @ccm node1 nodetool status
+
+cassandra-stop: .prepare-ccm
+ @echo "Stop cassandra cluster"
+ @ccm stop gocql_integration_test 2>/dev/null 1>&2 || true
+
+cassandra-remove: .prepare-ccm
+ @echo "Remove cassandra cluster"
+ @ccm remove gocql_integration_test 2>/dev/null 1>&2 || true
+ @rm -rf ${CCM_CONFIG_DIR}/gocql_integration_test || true
+
+test-integration: .prepare-cassandra-cluster
+ @echo "Run integration tests for proto ${TEST_CQL_PROTOCOL} on
cassandra ${CASSANDRA_VERSION}"
+ go test -v ${TEST_OPTS} -tags "${TEST_INTEGRATION_TAGS} gocql_debug"
-timeout=5m -proto=${TEST_CQL_PROTOCOL} -gocql.timeout=60s -runssl -rf=3
-clusterSize=3 -autowait=2000ms -compressor=${TEST_COMPRESSOR}
-gocql.cversion=${CASSANDRA_VERSION} -cluster=$$(ccm liveset) ./...
+
+test-integration-auth: .prepare-cassandra-cluster
+ @echo "Run auth integration tests for proto ${TEST_CQL_PROTOCOL} on
cassandra ${CASSANDRA_VERSION}"
+ go test -v -run=TestAuthentication -tags "${TEST_INTEGRATION_TAGS}
gocql_debug" -timeout=5m -proto=${TEST_CQL_PROTOCOL} -gocql.timeout=60s -runssl
-runauth -rf=3 -clusterSize=3 -autowait=2000ms -compressor=${TEST_COMPRESSOR}
-gocql.cversion=${CASSANDRA_VERSION} -cluster=$$(ccm liveset) ./...
+
+test-unit:
+ @echo "Run unit tests"
+ @go clean -testcache
+ go test -tags unit -timeout=5m -race ./...
+
+check:
+ @echo "Run go vet linter"
+ go vet --tags "unit all cassandra integration" ./...
+
+.prepare-java:
+ifeq ($(shell if [ -f ~/.sdkman/bin/sdkman-init.sh ]; then echo "installed";
else echo "not-installed"; fi), not-installed)
+ @$(MAKE) install-java
+else
+ @exit 0
+endif
+
+install-java:
+ @echo "Installing SDKMAN..."
+ @curl -s "https://get.sdkman.io"; | bash
+ @echo "sdkman_auto_answer=true" >> ~/.sdkman/etc/config
+ @( \
+ source ~/.sdkman/bin/sdkman-init.sh; \
+ export PATH=${PATH}:~/.sdkman/bin; \
+ echo "Installing Java versions..."; \
+ sdk install java 11.0.24-zulu; \
+ sdk install java 17.0.12-zulu; \
+ sdk default java 11.0.24-zulu; \
+ sdk use java 11.0.24-zulu; \
+ if [[ -n "${GITHUB_ENV}" ]]; then \
+ echo "JAVA11_HOME=$$JAVA_HOME_11_X64" >> ${GITHUB_ENV};
\
+ echo "JAVA17_HOME=$$JAVA_HOME_17_X64" >> ${GITHUB_ENV}; \
+ echo "JAVA_HOME=$$JAVA_HOME_11_X64" >> ${GITHUB_ENV}; \
+ echo "$$PATH" > ${GITHUB_PATH}; \
+ fi; \
+ )
+
+.prepare-ccm:
+ @ccm --help 2>/dev/null 1>&2; \
+ if [[ $$? -le 127 ]] && grep ${CCM_VERSION}
${CCM_CONFIG_DIR}/ccm-version 2>/dev/null 1>&2; then \
+ echo "CCM ${CCM_VERSION} is already installed"; \
+ else \
+ echo "Install CCM ${CCM_VERSION}"; \
+ pip install "git+https://github.com/riptano/ccm.git@${CCM_VERSION}";; \
+ mkdir ${CCM_CONFIG_DIR} 2>/dev/null 1>&2 || true; \
+ echo ${CCM_VERSION} > ${CCM_CONFIG_DIR}/ccm-version; \
+ fi
+
+install-ccm:
+ @echo "Install CCM ${CCM_VERSION}"
+ @pip install "git+https://github.com/riptano/ccm.git@${CCM_VERSION}";
+ @mkdir ${CCM_CONFIG_DIR} 2>/dev/null 1>&2 || true
+ @echo ${CCM_VERSION} > ${CCM_CONFIG_DIR}/ccm-version
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
