This is an automated email from the ASF dual-hosted git repository.
twice pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new 0f2de7d0 Enable CI for kvrocks2redis (#2175)
0f2de7d0 is described below
commit 0f2de7d0a275028cd9519a5ae22f0f97af1ffa2f
Author: Zakelly <[email protected]>
AuthorDate: Sun Apr 21 11:11:07 2024 +0800
Enable CI for kvrocks2redis (#2175)
Co-authored-by: 纪华裕 <[email protected]>
Co-authored-by: Twice <[email protected]>
Co-authored-by: hulk <[email protected]>
---
.github/workflows/kvrocks.yaml | 83 +++++++++++++++++++++++----
utils/kvrocks2redis/tests/populate-kvrocks.py | 11 +++-
2 files changed, 80 insertions(+), 14 deletions(-)
diff --git a/.github/workflows/kvrocks.yaml b/.github/workflows/kvrocks.yaml
index f2854143..b7767575 100644
--- a/.github/workflows/kvrocks.yaml
+++ b/.github/workflows/kvrocks.yaml
@@ -223,13 +223,21 @@ jobs:
path: |
~/local/bin/redis-cli
key: ${{ runner.os }}-${{ runner.arch }}-redis-cli
+ - name: Cache redis server
+ id: cache-redis-server
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/local/bin/redis-server
+ key: ${{ runner.os }}-${{ runner.arch }}-redis-server
- name: Install redis
- if: steps.cache-redis.outputs.cache-hit != 'true'
+ if: ${{ steps.cache-redis.outputs.cache-hit != 'true' ||
steps.cache-redis-server.outputs.cache-hit != 'true' }}
run: |
- curl -O https://download.redis.io/releases/redis-6.2.7.tar.gz
- tar -xzvf redis-6.2.7.tar.gz
+ curl -O https://download.redis.io/releases/redis-6.2.14.tar.gz
+ tar -xzvf redis-6.2.14.tar.gz
mkdir -p $HOME/local/bin
- pushd redis-6.2.7 && BUILD_TLS=yes make -j$NPROC redis-cli && mv
src/redis-cli $HOME/local/bin/ && popd
+ pushd redis-6.2.14 && BUILD_TLS=yes make -j$NPROC redis-cli && mv
src/redis-cli $HOME/local/bin/ && popd
+ pushd redis-6.2.14 && BUILD_TLS=yes make -j$NPROC redis-server && mv
src/redis-server $HOME/local/bin/ && popd
- uses: actions/checkout@v4
with:
@@ -256,7 +264,7 @@ jobs:
./x.py build -j$NPROC --unittest --compiler ${{ matrix.compiler }}
${{ matrix.without_jemalloc }} \
${{ matrix.without_luajit }} ${{ matrix.with_ninja }} ${{
matrix.with_sanitizer }} ${{ matrix.with_openssl }} \
${{ matrix.new_encoding }} ${{ matrix.with_speedb }} ${{
env.CMAKE_EXTRA_DEFS }}
-
+
- name: Build Kvrocks (SonarCloud)
if: ${{ matrix.sonarcloud }}
run: |
@@ -293,6 +301,30 @@ jobs:
fi
./x.py test go build $GOCASE_RUN_ARGS ${{ matrix.ignore_when_tsan}}
+ - name: Install redis-py
+ run: pip3 install redis==4.3.6
+
+ - name: Run kvrocks2redis Test
+ # Currently, when enabling Tsan/Asan or running in macOS 11/14, the
value mismatch in destination redis server.
+ # See https://github.com/apache/kvrocks/issues/2195.
+ if: ${{ !contains(matrix.name, 'Tsan') && !contains(matrix.name,
'Asan') && !startsWith(matrix.os, 'macos') }}
+ run: |
+ ulimit -c unlimited
+ export LSAN_OPTIONS="suppressions=$(realpath
./tests/lsan-suppressions)"
+ export TSAN_OPTIONS="suppressions=$(realpath
./tests/tsan-suppressions)"
+ $HOME/local/bin/redis-server --daemonize yes
+ mkdir -p kvrocks2redis-ci-data
+ ./build/kvrocks --dir `pwd`/kvrocks2redis-ci-data --pidfile
`pwd`/kvrocks.pid --daemonize yes
+ sleep 10s
+ echo -en "data-dir `pwd`/kvrocks2redis-ci-data\ndaemonize
yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >>
./kvrocks2redis-ci.conf
+ cat ./kvrocks2redis-ci.conf
+ ./build/kvrocks2redis -c ./kvrocks2redis-ci.conf
+ sleep 10s
+ python3 utils/kvrocks2redis/tests/populate-kvrocks.py --password=""
--flushdb=true
+ sleep 10s
+ ps aux
+ python3 utils/kvrocks2redis/tests/check_consistency.py
--src_password=""
+
- name: Find reports and crashes
if: always()
run: |
@@ -397,7 +429,7 @@ jobs:
if: ${{ startsWith(matrix.image, 'centos') }}
run: |
yum install -y centos-release-scl-rh
- yum install -y devtoolset-11 python3 autoconf automake wget git gcc
gcc-c++
+ yum install -y devtoolset-11 python3 python3-pip autoconf automake
wget git gcc gcc-c++
echo "NPROC=$(nproc)" >> $GITHUB_ENV
mv /usr/bin/gcc /usr/bin/gcc-4.8.5
ln -s /opt/rh/devtoolset-11/root/bin/gcc /usr/bin/gcc
@@ -408,13 +440,13 @@ jobs:
if: ${{ startsWith(matrix.image, 'archlinux') }}
run: |
pacman -Syu --noconfirm
- pacman -Sy --noconfirm autoconf automake python3 git wget which
cmake make gcc
+ pacman -Sy --noconfirm autoconf automake python3 python-redis git
wget which cmake make gcc
echo "NPROC=$(nproc)" >> $GITHUB_ENV
- name: Setup openSUSE
if: ${{ startsWith(matrix.image, 'opensuse') }}
run: |
- zypper install -y gcc11 gcc11-c++ make wget git autoconf automake
python3 curl tar gzip cmake go
+ zypper install -y gcc11 gcc11-c++ make wget git autoconf automake
python3 python3-pip curl tar gzip cmake go
update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-11 100
update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-11 100
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
@@ -429,13 +461,22 @@ jobs:
~/local/bin/redis-cli
key: ${{ matrix.image }}-redis-cli
+ - name: Cache redis server
+ id: cache-redis-server
+ uses: actions/cache@v3
+ with:
+ path: |
+ ~/local/bin/redis-server
+ key: ${{ matrix.image }}-redis-server
+
- name: Install redis
- if: steps.cache-redis.outputs.cache-hit != 'true'
+ if: ${{ steps.cache-redis.outputs.cache-hit != 'true' ||
steps.cache-redis-server.outputs.cache-hit != 'true' }}
run: |
- curl -O https://download.redis.io/releases/redis-6.2.7.tar.gz
- tar -xzvf redis-6.2.7.tar.gz
+ curl -O https://download.redis.io/releases/redis-6.2.14.tar.gz
+ tar -xzvf redis-6.2.14.tar.gz
mkdir -p $HOME/local/bin
- pushd redis-6.2.7 && USE_JEMALLOC=no make -j$NPROC redis-cli && mv
src/redis-cli $HOME/local/bin/ && popd
+ pushd redis-6.2.14 && USE_JEMALLOC=no make -j$NPROC redis-cli && mv
src/redis-cli $HOME/local/bin/ && popd
+ pushd redis-6.2.14 && USE_JEMALLOC=no make -j$NPROC redis-server &&
mv src/redis-server $HOME/local/bin/ && popd
- name: Install cmake
if: ${{ startsWith(matrix.image, 'centos') }}
@@ -465,6 +506,24 @@ jobs:
GOCASE_RUN_ARGS=""
./x.py test go build $GOCASE_RUN_ARGS
+ - name: Install redis-py
+ if: ${{ !startsWith(matrix.image, 'archlinux') }} # already installed
+ run: pip3 install redis==4.3.6
+
+ - name: Run kvrocks2redis Test
+ run: |
+ $HOME/local/bin/redis-server --daemonize yes
+ mkdir -p kvrocks2redis-ci-data
+ ./build/kvrocks --dir `pwd`/kvrocks2redis-ci-data --pidfile
`pwd`/kvrocks.pid --daemonize yes
+ sleep 10s
+ echo -en "data-dir `pwd`/kvrocks2redis-ci-data\ndaemonize
yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >>
./kvrocks2redis-ci.conf
+ cat ./kvrocks2redis-ci.conf
+ ./build/kvrocks2redis -c ./kvrocks2redis-ci.conf
+ sleep 10s
+ python3 utils/kvrocks2redis/tests/populate-kvrocks.py --password=""
--flushdb=true
+ sleep 10s
+ python3 utils/kvrocks2redis/tests/check_consistency.py
--src_password=""
+
required:
if: always()
name: Required
diff --git a/utils/kvrocks2redis/tests/populate-kvrocks.py
b/utils/kvrocks2redis/tests/populate-kvrocks.py
index 8883a0df..fefae86a 100644
--- a/utils/kvrocks2redis/tests/populate-kvrocks.py
+++ b/utils/kvrocks2redis/tests/populate-kvrocks.py
@@ -146,8 +146,10 @@ def run_test(client, cases : list):
print('******* Some case test fail *******')
for cmd in fails:
print(cmd)
+ return False
else:
print('All case passed.')
+ return True
if __name__ == '__main__':
@@ -155,5 +157,10 @@ if __name__ == '__main__':
client = redis.Redis(host=args.host, port=args.port,
decode_responses=True, password=args.password)
if args.flushdb:
client.flushdb()
- run_test(client, PopulateCases)
- run_test(client, AppendCases)
+ succ = True
+ if not run_test(client, PopulateCases):
+ succ = False
+ if not run_test(client, AppendCases):
+ succ = False
+ if not succ:
+ raise AssertionError("Test failed. See details above.")
