This is an automated email from the ASF dual-hosted git repository.
wutao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new dcde890e chore: Merge repo pegasus-kv/pegasus-docker (#944)
dcde890e is described below
commit dcde890e7655ff8b8403535a578c0f5f9f8f080e
Author: Yingchun Lai <[email protected]>
AuthorDate: Fri Apr 15 16:36:50 2022 +0800
chore: Merge repo pegasus-kv/pegasus-docker (#944)
---
.circleci/config.yml | 26 +++
.github/build-push-env-docker.yml | 68 ++++++
.github/pegasus-regular-build.yml | 93 ++++++++
.github/thirdparty-regular-push.yml | 106 +++++++++
docker/README.md | 64 ++++++
docker/bcc-centos7/Dockerfile | 52 +++++
docker/ci-env/Dockerfile | 33 +++
docker/clang-format-3.9/Dockerfile | 28 +++
docker/pegasus-build-env/README.md | 21 ++
docker/pegasus-build-env/centos7/Dockerfile | 70 ++++++
docker/pegasus-build-env/ubuntu1604/Dockerfile | 65 ++++++
docker/pegasus-build-env/ubuntu1804/Dockerfile | 72 ++++++
docker/pegasus-build-env/ubuntu2004/Dockerfile | 72 ++++++
docker/pegasus-docker-compose/README.md | 137 ++++++++++++
docker/pegasus-docker-compose/build_docker.sh | 58 +++++
docker/pegasus-docker-compose/clear_onebox.sh | 38 ++++
docker/pegasus-docker-compose/cluster_args.sh | 52 +++++
docker/pegasus-docker-compose/config.min.ini | 242 +++++++++++++++++++++
docker/pegasus-docker-compose/docker-compose.yml | 36 +++
.../image_for_prebuilt_bin/Dockerfile | 33 +++
.../image_for_prebuilt_bin/entrypoint.sh | 25 +++
docker/pegasus-docker-compose/prepare.sh | 93 ++++++++
docker/pegasus-docker-compose/start_onebox.sh | 45 ++++
docker/thirdparties-bin/Dockerfile | 38 ++++
docker/thirdparties-src/Dockerfile | 43 ++++
25 files changed, 1610 insertions(+)
diff --git a/.circleci/config.yml b/.circleci/config.yml
new file mode 100644
index 00000000..6866e73f
--- /dev/null
+++ b/.circleci/config.yml
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+version: 2.1
+orbs:
+ docker: circleci/[email protected]
+workflows:
+ lint:
+ jobs:
+ - docker/hadolint:
+ dockerfiles:
'docker/bcc-centos7/Dockerfile:docker/ci-env/Dockerfile:docker/clang-format-3.9/Dockerfile:docker/pegasus-build-env/centos7/Dockerfile:docker/pegasus-build-env/ubuntu1604/Dockerfile:docker/pegasus-build-env/ubuntu1804/Dockerfile:docker/pegasus-build-env/ubuntu2004/Dockerfile:docker/pegasus-docker-compose/image_for_prebuilt_bin/Dockerfile:docker/thirdparties-bin/Dockerfile:docker/thirdparties-src/Dockerfile'
+ ignore-rules:
'DL3033,DL3013,DL3059,SC2086,DL3003,SC2164,DL3008,DL3007,DL3006'
diff --git a/.github/build-push-env-docker.yml
b/.github/build-push-env-docker.yml
new file mode 100644
index 00000000..4711fcda
--- /dev/null
+++ b/.github/build-push-env-docker.yml
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+name: BuildCompilationEnvDocker-build and publish multi compilation os env
+
+on:
+ push:
+ paths:
+ - 'docker/pegasus-build-env/**'
+ - '.github/workflows/build-push-env-docker.yml'
+
+ # for manually triggering workflow
+ workflow_dispatch:
+
+jobs:
+ build_compilation_env_docker_images:
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ dockertag:
+ - ubuntu1604
+ - ubuntu1804
+ - ubuntu2004
+ #- centos6 is EOL
+ - centos7
+ steps:
+ -
+ name: Checkout
+ uses: actions/checkout@v2
+ -
+ name: Set up QEMU
+ uses: docker/setup-qemu-action@v1
+ -
+ name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+ -
+ name: Login to DockerHub
+ uses: docker/login-action@v1
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ -
+ name: Build and push
+ uses: docker/build-push-action@v2
+ with:
+ context: .
+ file: ./docker/pegasus-build-env/${{ matrix.dockertag }}/Dockerfile
+ push: true
+ tags: |
+ apachepegasus/build-env:${{ matrix.dockertag }}
+ -
+ name: Image digest
+ run: echo ${{ steps.docker_build.outputs.digest }}
diff --git a/.github/pegasus-regular-build.yml
b/.github/pegasus-regular-build.yml
new file mode 100644
index 00000000..23d58855
--- /dev/null
+++ b/.github/pegasus-regular-build.yml
@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+name: BuildPegasusRegularly-build pegasus and rdsn on different env every day
+
+on:
+ push:
+ paths:
+ - '.github/workflows/pegasus-regular-build.yml'
+
+ # for manually triggering workflow
+ workflow_dispatch:
+
+ # run for every day 2am UTC+8(Beijing)
+ schedule:
+ - cron: '0 18 */1 * *'
+
+jobs:
+ build_docker_images:
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ os:
+ - ubuntu1604
+ - ubuntu1804
+ - ubuntu2004
+ #- centos6 is EOL
+ - centos7
+ compiler-family:
+ - gcc
+ include:
+ - compiler-family: clang
+ compiler: "clang-10,clang++-10"
+ os: ubuntu2004
+ - compiler-family: clang
+ compiler: "clang-9,clang++-9"
+ os: ubuntu1804
+ container:
+ image: apachepegasus/thirdparties-bin:${{ matrix.os }}
+ defaults:
+ run:
+ working-directory: /root/incubator-pegasus
+ steps:
+ - name: Clone Apache Pegasus Source
+ working-directory: /root
+ run: |
+ git clone --recursive --depth=1
https://github.com/apache/incubator-pegasus.git
+ cd incubator-pegasus
+ -
+ name: Unpack prebuilt third-parties
+ run: unzip /root/thirdparties-bin.zip -d ./rdsn/thirdparty
+ -
+ name: Compilation rdsn on GCC
+ if: ${{ matrix.compiler-family == 'gcc' }}
+ run: |
+ cd ./rdsn
+ ./run.sh build -c --skip_thirdparty
+ -
+ name: Compilation pegasus on GCC
+ if: ${{ matrix.compiler-family == 'gcc' }}
+ run: ./run.sh build -c --skip_thirdparty
+ -
+ name: Compilation rdsn on Clang
+ if: ${{ matrix.compiler-family == 'clang' }}
+ env:
+ COMPILER: ${{ matrix.compiler }}
+ run: |
+ cd ./rdsn
+ ./run.sh build --compiler $COMPILER --skip_thirdparty
+ -
+ name: Compilation pegasus on Clang
+ if: ${{ matrix.compiler-family == 'clang' }}
+ env:
+ COMPILER: ${{ matrix.compiler }}
+ run: ./run.sh build --compiler $COMPILER --skip_thirdparty
+ -
+ name: Packaging Server
+ run: ./run.sh pack_server
diff --git a/.github/thirdparty-regular-push.yml
b/.github/thirdparty-regular-push.yml
new file mode 100644
index 00000000..96c716af
--- /dev/null
+++ b/.github/thirdparty-regular-push.yml
@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+name: BuildThirdpartyDockerRegularly-build and publish thirdparty every week
+
+on:
+ push:
+ paths:
+ - 'docker/thirdparties-src/**'
+ - 'docker/thirdparties-bin/**'
+ - '.github/workflows/thirdparty-regular-push.yml'
+
+ # for manually triggering workflow
+ workflow_dispatch:
+
+ # run for every week 2am UTC+8(Beijing)
+ schedule:
+ - cron: '0 18 * * 1'
+
+jobs:
+ build_push_src_docker_images:
+ runs-on: ubuntu-latest
+ steps:
+ -
+ uses: actions/checkout@v2
+ -
+ name: Set up QEMU
+ uses: docker/setup-qemu-action@v1
+ -
+ name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+ -
+ name: Login to DockerHub
+ uses: docker/login-action@v1
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ -
+ name: Build and push
+ uses: docker/build-push-action@v2
+ with:
+ context: .
+ file: ./docker/thirdparties-src/Dockerfile
+ push: true
+ tags: |
+ apachepegasus/thirdparties-src
+ build-args: |
+ REGISTRY_REPO=apachepegasus
+ -
+ name: Image digest
+ run: echo ${{ steps.docker_build.outputs.digest }}
+
+ build_push_bin_docker_images:
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ osversion:
+ - ubuntu1604
+ - ubuntu1804
+ - ubuntu2004
+ #- centos6 is EOL
+ - centos7
+ steps:
+ -
+ uses: actions/checkout@v2
+ -
+ name: Set up QEMU
+ uses: docker/setup-qemu-action@v1
+ -
+ name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+ -
+ name: Login to DockerHub
+ uses: docker/login-action@v1
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ -
+ name: Build and push
+ uses: docker/build-push-action@v2
+ with:
+ context: .
+ file: ./docker/thirdparties-bin/Dockerfile
+ push: true
+ tags: |
+ apachepegasus/thirdparties-bin:${{ matrix.osversion }}
+ build-args: |
+ OS_VERSION=${{ matrix.osversion }}
+ -
+ name: Image digest
+ run: echo ${{ steps.docker_build.outputs.digest }}
diff --git a/docker/README.md b/docker/README.md
new file mode 100644
index 00000000..cddd4093
--- /dev/null
+++ b/docker/README.md
@@ -0,0 +1,64 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+# Pegasus Docker
+
+This project maintains the stuff you can use from building Pegasus docker
images,
+to deploying a standalone cluster of Pegasus containers on your local machine.
+
+## Workflows
+
+
+
+
+
+
+
+## pegasus-build-env
+
+Building environment for Pegasus compilation.
+
+Github Actions automatically rebuilds and publishes build-env for every commit.
+
+- `apachepegasus/build-env:centos7`
+- `apachepegasus/build-env:ubuntu1604`
+- `apachepegasus/build-env:ubuntu1804`
+- `apachepegasus/build-env:ubuntu2004`
+
+DockerHub: https://hub.docker.com/r/apachepegasus/build-env
+
+The How-to-use Manual is at:
http://pegasus.apache.org/docs/build/compile-by-docker/
+
+## thirdparties-src
+
+This image is to eliminate extra downloading of third-party sources of Pegasus.
+It packages the downloaded sources into a zip in the container, so that
+other repos can easily extract third-parties from the container (via `docker
cp`),
+without downloading from the cloud object storage.
+
+- `apachepegasus/thirdparties-src`
+
+## thirdparties-bin
+
+This is a Docker image for Pegasus unit-testing. It prebuilts the thirdparty
libraries,
+so jobs based on this image can skip building third-parties.
+
+- `apachepegasus/thirdparties-bin:centos7`
+- `apachepegasus/thirdparties-bin:ubuntu1604`
+- `apachepegasus/thirdparties-bin:ubuntu1804`
+- `apachepegasus/thirdparties-bin:ubuntu2004`
diff --git a/docker/bcc-centos7/Dockerfile b/docker/bcc-centos7/Dockerfile
new file mode 100644
index 00000000..c6129662
--- /dev/null
+++ b/docker/bcc-centos7/Dockerfile
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM centos:7.5.1804 as builder
+
+LABEL maintainer=wutao
+
+RUN yum install -y epel-release \
+ && yum update -y \
+ && yum groupinstall -y "Development tools" \
+ && yum install -y elfutils-libelf-devel git bison flex ncurses-devel python3
\
+ && yum install -y luajit luajit-devel \
+ && yum -y install centos-release-scl \
+ && yum-config-manager --enable rhel-server-rhscl-7-rpms \
+ && yum install -y devtoolset-7 llvm-toolset-7 llvm-toolset-7-llvm-devel
llvm-toolset-7-llvm-static llvm-toolset-7-clang-devel \
+ && yum clean all \
+ && rm -rf /var/cache/yum;
+
+RUN pip3 install --no-cache-dir cmake
+
+# install results to /root/bcc
+RUN git clone https://github.com/iovisor/bcc.git \
+ && cd bcc \
+ && source scl_source enable devtoolset-7 llvm-toolset-7 \
+ && cmake -B build/ . \
+ && cmake --build build/ -j $(($(nproc)/2+1)) \
+ && cmake --install build/ --prefix $HOME/bcc \
+ && rm -rf /bcc
+
+FROM centos:7.5.1804
+
+COPY --from=builder /root/bcc /bcc-pkg/bcc
+COPY --from=builder /opt/rh/llvm-toolset-7 /bcc-pkg/llvm-toolset-7
+COPY --from=builder /usr/lib/python2.7/site-packages/ /bcc-pkg/site-packages/
+
+ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/bcc-pkg/bcc/lib64/"
+ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/bcc-pkg/llvm-toolset-7/root/lib64"
+ENV PYTHONPATH="${PYTHONPATH}:/bcc-pkg/site-packages"
diff --git a/docker/ci-env/Dockerfile b/docker/ci-env/Dockerfile
new file mode 100644
index 00000000..c7d49d1f
--- /dev/null
+++ b/docker/ci-env/Dockerfile
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM apachepegasus/build-env:centos7
+
+WORKDIR /root
+
+RUN yum -y install ccache zip nmap-ncat; \
+ yum clean all; \
+ rm -rf /var/cache/yum;
+
+RUN git clone --depth=1 https://github.com/XiaoMi/rdsn.git; \
+ cd rdsn/thirdparty; \
+ mkdir -p build; \
+ cmake -DCMAKE_BUILD_TYPE=Release -B build/ .; \
+ cmake --build build/ -j $(($(nproc)/2+1)); \
+ zip -r ~/pegasus-thirdparty-output.zip output/ build/Source/rocksdb/cmake
build/Source/http-parser build/Source/hadoop build/Download/zookeeper; \
+ cd ~; \
+ rm -rf rdsn;
diff --git a/docker/clang-format-3.9/Dockerfile
b/docker/clang-format-3.9/Dockerfile
new file mode 100644
index 00000000..2ebea2de
--- /dev/null
+++ b/docker/clang-format-3.9/Dockerfile
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM ubuntu:18.04
+
+LABEL maintainer=wutao
+
+RUN sed -i 's/archive.ubuntu.com/mirrors.aliyun.com/' /etc/apt/sources.list \
+ && apt-get update -y \
+ && apt-get install --no-install-recommends -y software-properties-common \
+ && add-apt-repository ppa:git-core/ppa \
+ && apt-get update -y \
+ && apt-get install --no-install-recommends -y clang-format-3.9 git \
+ && rm -rf /var/lib/apt/lists/*
diff --git a/docker/pegasus-build-env/README.md
b/docker/pegasus-build-env/README.md
new file mode 100644
index 00000000..89a19a1d
--- /dev/null
+++ b/docker/pegasus-build-env/README.md
@@ -0,0 +1,21 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+# pegasus-build-env
+
+This is the building environment of pegasus. Please read the manual here:
http://pegasus.apache.org/docs/build/compile-by-docker/.
diff --git a/docker/pegasus-build-env/centos7/Dockerfile
b/docker/pegasus-build-env/centos7/Dockerfile
new file mode 100644
index 00000000..6655208a
--- /dev/null
+++ b/docker/pegasus-build-env/centos7/Dockerfile
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM centos:7.5.1804
+
+LABEL maintainer=wutao
+
+RUN yum -y install centos-release-scl \
+scl-utils \
+epel-release; \
+yum -y install devtoolset-7-gcc \
+devtoolset-7-gcc-c++ \
+java-1.8.0-openjdk-devel.x86_64 \
+python3 \
+automake \
+autoconf \
+make \
+libtool \
+git \
+file \
+wget \
+ccache \
+nmap-ncat \
+zip \
+gdb \
+vim \
+unzip \
+which \
+openssl-devel \
+libaio-devel \
+snappy-devel \
+bzip2-devel \
+zlib \
+zlib-devel \
+libzstd-devel \
+lz4-devel \
+bison \
+flex \
+krb5-devel \
+cyrus-sasl-devel \
+patch; \
+yum clean all; \
+rm -rf /var/cache/yum;
+
+RUN pip3 install --no-cache-dir cmake
+
+RUN wget --progress=dot:giga
https://archive.apache.org/dist/maven/maven-3/3.8.3/binaries/apache-maven-3.8.3-bin.tar.gz
-P /opt/maven \
+ && cd /opt/maven \
+ && tar -zxf apache-maven-3.8.3-bin.tar.gz \
+ && rm apache-maven-3.8.3-bin.tar.gz
+
+ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.322.b06-1.el7_9.x86_64
+ENV PATH="/opt/rh/devtoolset-7/root/bin/:${PATH}"
+ENV PATH="/opt/maven/apache-maven-3.8.3/bin:${PATH}"
+
+WORKDIR /root/apache
diff --git a/docker/pegasus-build-env/ubuntu1604/Dockerfile
b/docker/pegasus-build-env/ubuntu1604/Dockerfile
new file mode 100644
index 00000000..937ccd3b
--- /dev/null
+++ b/docker/pegasus-build-env/ubuntu1604/Dockerfile
@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM ubuntu:16.04
+
+LABEL maintainer=wutao
+
+ENV TZ=Asia/Shanghai
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN apt-get update -y; \
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ software-properties-common \
+ openjdk-8-jdk \
+ python3-pip \
+ libaio-dev \
+ libsnappy-dev \
+ libbz2-dev \
+ libzstd-dev \
+ liblz4-dev \
+ zlib1g \
+ zlib1g.dev \
+ patch \
+ netcat \
+ wget \
+ ccache \
+ git \
+ curl \
+ zip \
+ unzip \
+ gdb \
+ vim \
+ automake \
+ libtool \
+ libssl-dev \
+ bison \
+ libkrb5-dev \
+ libsasl2-dev \
+ maven \
+ flex; \
+ rm -rf /var/lib/apt/lists/*
+
+RUN add-apt-repository ppa:git-core/ppa -y; \
+ apt-get update -y; \
+ apt-get install git pkg-config -y --no-install-recommends; \
+ rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install --no-cache-dir cmake
+
+WORKDIR /root/apache
diff --git a/docker/pegasus-build-env/ubuntu1804/Dockerfile
b/docker/pegasus-build-env/ubuntu1804/Dockerfile
new file mode 100644
index 00000000..96a20f88
--- /dev/null
+++ b/docker/pegasus-build-env/ubuntu1804/Dockerfile
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM ubuntu:18.04
+
+LABEL maintainer=wutao
+
+ENV TZ=Asia/Shanghai
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN apt-get update -y; \
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ software-properties-common \
+ clang-9 \
+ openjdk-8-jdk \
+ python3-pip \
+ libaio-dev \
+ libsnappy-dev \
+ libbz2-dev \
+ libzstd-dev \
+ liblz4-dev \
+ zlib1g \
+ zlib1g.dev \
+ patch \
+ netcat \
+ wget \
+ ccache \
+ git \
+ curl \
+ zip \
+ unzip \
+ gdb \
+ vim \
+ automake \
+ libtool \
+ libssl-dev \
+ bison \
+ libkrb5-dev \
+ libsasl2-dev \
+ maven \
+ flex; \
+ rm -rf /var/lib/apt/lists/*
+
+RUN add-apt-repository ppa:git-core/ppa -y; \
+ apt-get update -y; \
+ apt-get install git -y --no-install-recommends; \
+ apt-get install pkg-config -y --no-install-recommends; \
+ rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install --no-cache-dir cmake
+
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+ENV CLASSPATH=$JAVA_HOME/lib/
+ENV PATH=$JAVA_HOME/bin:$PATH
+ENV LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:$LD_LIBRARY_PATH
+
+WORKDIR /root/apache
diff --git a/docker/pegasus-build-env/ubuntu2004/Dockerfile
b/docker/pegasus-build-env/ubuntu2004/Dockerfile
new file mode 100644
index 00000000..69401c94
--- /dev/null
+++ b/docker/pegasus-build-env/ubuntu2004/Dockerfile
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM ubuntu:20.04
+
+LABEL maintainer=wutao
+
+ENV TZ=Asia/Shanghai
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN apt-get update -y; \
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ software-properties-common \
+ clang-10 \
+ openjdk-8-jdk \
+ python3-pip \
+ libaio-dev \
+ libsnappy-dev \
+ libbz2-dev \
+ libzstd-dev \
+ liblz4-dev \
+ zlib1g \
+ zlib1g.dev \
+ patch \
+ netcat \
+ wget \
+ ccache \
+ git \
+ curl \
+ zip \
+ unzip \
+ gdb \
+ vim \
+ automake \
+ libtool \
+ libssl-dev \
+ bison \
+ libkrb5-dev \
+ libsasl2-dev \
+ maven \
+ flex; \
+ rm -rf /var/lib/apt/lists/*
+
+RUN add-apt-repository ppa:git-core/ppa -y; \
+ apt-get update -y; \
+ apt-get install git -y --no-install-recommends; \
+ apt-get install pkg-config -y --no-install-recommends; \
+ rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install --no-cache-dir cmake
+
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+ENV CLASSPATH=$JAVA_HOME/lib/
+ENV PATH=$JAVA_HOME/bin:$PATH
+ENV LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:$LD_LIBRARY_PATH
+
+WORKDIR /root/apache
diff --git a/docker/pegasus-docker-compose/README.md
b/docker/pegasus-docker-compose/README.md
new file mode 100644
index 00000000..7d84672d
--- /dev/null
+++ b/docker/pegasus-docker-compose/README.md
@@ -0,0 +1,137 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+# Pegasus with Docker Compose
+
+This is a docker-compose solution to bootstrap a Pegasus cluster
+on your machine.
+
+## HOW TO
+
+### Docker image
+
+Before setting up the dockers, you should first have a docker image of a
specfic version of Pegasus.
+
+Run compilation and package your binaries:
+
+```sh
+cd /your/local/apache-pegasus-source
+./run.sh build -c # Compiling the code.
+./run.sh pack_server # Packaging the binaries of Pegasus server.
+```
+
+Then build the image:
+
+```
+cd /your/local/pegasus-docker/pegasus-docker-compose
+./build_docker.sh /your/local/apache-pegasus-source
+```
+
+You will have a docker image called "pegasus:latest" right now built on you
machine. Check it out:
+
+```sh
+docker images
+```
+
+### Set up a docker cluster
+
+To start a cluster:
+
+```sh
+./start_onebox.sh
+```
+
+To start multiple clusters, you should reconfigure the `cluster_args.sh` by
using a different set of:
+
+- `CLUSTER_NAME`
+- `META_IP_PREFIX`
+- `META_PORT`.
+
+For example `onebox2`, `173.21.0` and `35601`.
+
+Use `docker ps` command to show all the running dockers. And if some docker
failed unexpectedly,
+use `docker logs {DOCKER_ID}` to dump the error logs of it.
+
+### Configuration
+
+`pegasus-docker/pegasus-docker-compose/config.min.ini` is the
master-branch-Pegasus config. It may be not available
+for Pegasus in a previously released version.
+
+```sh
+cd /your/local/pegasus-docker
+git checkout 2.1.0
+```
+
+If you want to set up a previously released Pegasus cluster, please use the
scripts from the specific branch.
+
+### Cleanup a docker cluster
+
+```sh
+./clear_onebox.sh
+```
+
+This script will kill the cluster processes, remove the virtual network and
finally remove the data directory.
+
+## Using pumba to inject faults to Pegasus
+
+[pumba](https://github.com/alexei-led/pumba) is a chaos testing and network
emulation tool for Docker. We can use it
+for fault injection of Pegasus, testing Pegasus's stability under various
conditions
+that pumba provides, including:
+
+- docker pause
+- network loss
+- network corrupt
+- network delay
+
+To establish the testing environment, first set up a 3-replica, 2-meta onebox
cluster. Modify your cluster_arg.sh file like this:
+
+```sh
+export META_COUNT=2
+export REPLICA_COUNT=3
+```
+
+```sh
+> ./start_onebox.sh
+
+ Name Command State
Ports
+---------------------------------------------------------------------------------------------------------------------
+onebox-docker_meta1_1 /entrypoint.sh meta Up
0.0.0.0:34601->34601/tcp
+onebox-docker_meta2_1 /entrypoint.sh meta Up
0.0.0.0:34602->34601/tcp
+onebox-docker_replica1_1 /entrypoint.sh replica Up
0.0.0.0:32774->34801/tcp
+onebox-docker_replica2_1 /entrypoint.sh replica Up
0.0.0.0:32776->34801/tcp
+onebox-docker_replica3_1 /entrypoint.sh replica Up
0.0.0.0:32775->34801/tcp
+onebox-docker_zookeeper_1 /docker-entrypoint.sh zkSe ... Up
0.0.0.0:32777->2181/tcp, 2888/tcp, 3888/tcp
+```
+
+Run your client-side tool (Pegasus-YCSB, e.g.) to keep requesting to Pegasus
service.
+
+```sh
+~/pegasus-YCSB/pegasus-YCSB-0.12.0-SNAPSHOT
+
+> ./start.sh load
+```
+
+Begin injecting faults to onebox-docker:
+
+```sh
+pumba pause --duration=1h onebox-docker_replica1_1
+
+pumba netem --duration=1h --tc-image 'gaiadocker/iproute2' loss --percent 100
onebox-docker_replica1_1
+```
+
+Ensure your Pegasus service failovers normally, with no unexpectedly failed
requests and error logs from your client.
diff --git a/docker/pegasus-docker-compose/build_docker.sh
b/docker/pegasus-docker-compose/build_docker.sh
new file mode 100755
index 00000000..f665187c
--- /dev/null
+++ b/docker/pegasus-docker-compose/build_docker.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#!/bin/bash
+
+# Usage:
+# ./build_docker.sh /your/local/apache-pegasus-source
+
+set -e
+
+if [[ $# -ne 1 ]]; then
+ echo "ERROR: must specify /your/local/apache-pegasus-source"
+ exit 1
+fi
+
+# ROOT is where the script is.
+ROOT=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")
+
+# PEGASUS_ROOT is the absolute path of apache-pegasus-source
+PEGASUS_ROOT="$(readlink -f "$1")"
+if [[ ! -f "${PEGASUS_ROOT}"/PACKAGE ]]; then
+ echo "ERROR: no such file ${PEGASUS_ROOT}/PACKAGE"
+ exit 1
+fi
+SERVER_PKG_NAME=$(cat "${PEGASUS_ROOT}"/PACKAGE)
+if [[ ! -f "${PEGASUS_ROOT}/${SERVER_PKG_NAME}.tar.gz" ]]; then
+ echo "Failed to find package ${SERVER_PKG_NAME}.tar.gz in
${PEGASUS_ROOT}"
+ exit 1
+else
+ echo "Found package ${PEGASUS_ROOT}/${SERVER_PKG_NAME}.tar.gz"
+fi
+
+###
+# configurable
+IMAGE_NAME=pegasus:latest
+###
+
+echo "Building image ${IMAGE_NAME}"
+cd "${ROOT}"/image_for_prebuilt_bin || exit 1
+
+cp "${PEGASUS_ROOT}/${SERVER_PKG_NAME}.tar.gz" .
+docker build --build-arg SERVER_PKG_NAME="${SERVER_PKG_NAME}" -t ${IMAGE_NAME}
.
+rm "${SERVER_PKG_NAME}.tar.gz"
diff --git a/docker/pegasus-docker-compose/clear_onebox.sh
b/docker/pegasus-docker-compose/clear_onebox.sh
new file mode 100755
index 00000000..2ab29127
--- /dev/null
+++ b/docker/pegasus-docker-compose/clear_onebox.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#!/usr/bin/env bash
+
+cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
+
+source cluster_args.sh
+
+if ! [[ -d "${DOCKER_DIR}" ]]; then
+ echo "Cleared ${DOCKER_DIR} already"
+ exit 0
+fi
+cd "${DOCKER_DIR}" || exit 1
+pwd
+
+docker-compose kill
+docker-compose rm -f -v
+docker network prune -f
+
+cd "${ROOT}" || exit 1
+
+sudo rm -rf "${DOCKER_DIR}"
diff --git a/docker/pegasus-docker-compose/cluster_args.sh
b/docker/pegasus-docker-compose/cluster_args.sh
new file mode 100755
index 00000000..6570138a
--- /dev/null
+++ b/docker/pegasus-docker-compose/cluster_args.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#!/usr/bin/env bash
+
+# Configure the following variables to customize the docker cluster. #
+
+# The ip prefix for each meta.
+# Meta-x's ip address is 172.21.0.1{x}:34601.
+# For exmaple, Meta1's address is 172.21.0.11:34601.
+export META_IP_PREFIX=172.21.0
+
+# The exported port of pegasus meta-server.
+# Please ensure this port is not occupied by other programs.
+export META_PORT=34601
+
+# Different clusters are isolated by their cluster name and the META_IP_PREFIX.
+export CLUSTER_NAME=onebox
+
+export IMAGE_NAME='pegasus:latest'
+
+# allow_non_idempotent_write = true
+# for jepsen test this option must be enabled.
+export IDEMPOTENT=true
+export META_COUNT=1 # Number of meta instances.
+export REPLICA_COUNT=1 # Number of replica instances.
+
+# Config End #
+##############
+# The following are constants.
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
+export ROOT
+# Where docker onebox resides. If the cluster name is 'onebox', all nodes will
mount their data
+# upon the directory ./onebox-docker.
+export DOCKER_DIR=${ROOT}/${CLUSTER_NAME}-docker
diff --git a/docker/pegasus-docker-compose/config.min.ini
b/docker/pegasus-docker-compose/config.min.ini
new file mode 100644
index 00000000..ceb9a33d
--- /dev/null
+++ b/docker/pegasus-docker-compose/config.min.ini
@@ -0,0 +1,242 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements. See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership. The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License. You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing,
+; software distributed under the License is distributed on an
+; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+; KIND, either express or implied. See the License for the
+; specific language governing permissions and limitations
+; under the License.
+[apps..default]
+ run = true
+ count = 1
+
+[apps.meta]
+ type = meta
+ name = meta
+ ports = %{meta.port}
+ pools =
THREAD_POOL_DEFAULT,THREAD_POOL_META_SERVER,THREAD_POOL_META_STATE,THREAD_POOL_FD,THREAD_POOL_DLOCK,THREAD_POOL_BLOCK_SERVICE
+
+[apps.replica]
+ type = replica
+ name = replica
+ ports = 34801
+ pools =
THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_INGESTION,THREAD_POOL_SLOG,THREAD_POOL_PLOG
+
+[apps.collector]
+ name = collector
+ type = collector
+ ports = 34101
+ pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION
+
+[core]
+ tool = nativerun
+ toollets = profiler
+ enable_default_app_mimic = true
+ logging_start_level = LOG_LEVEL_DEBUG
+
+[block_service.local_service]
+ type = local_service
+ args = ../block_service/local_service
+
+[tools.simple_logger]
+ short_header = false
+ stderr_start_level = LOG_LEVEL_ERROR
+
+[threadpool..default]
+ worker_count = 4
+ worker_priority = THREAD_xPRIORITY_NORMAL
+ partitioned = false
+
+[threadpool.THREAD_POOL_DEFAULT]
+ name = default
+ # The worker count in THREAD_POOL_DEFAULT must be >= 5.
+ # Because in info collector server, there are four timer
tasks(LPC_PEGASUS_APP_STAT_TIMER, LPC_PEGASUS_STORAGE_SIZE_STAT_TIMER,
+ # LPC_DETECT_AVAILABLE and LPC_PEGASUS_CAPACITY_UNIT_STAT_TIMER). Each of
these timer tasks occupies a thread in THREAD_POOL_DEFAULT.
+ # Each of these timer tasks calls remote procedure to meta server(which
produce a callback), and waits for the rpc's callback to execute.
+ # If the worker_count <= 4, all of these threads are occupied by these timer
tasks. so their rpc's callbacks can't get a thread to run.
+ # it comes to be a deadlock(timer task wait for rpc's callback to execute,
and rpc's callback wait for the timer task to release the thread).
+ worker_count = 5
+
+[threadpool.THREAD_POOL_REPLICATION]
+ name = replica
+ partitioned = true
+ worker_count = 2
+
+[threadpool.THREAD_POOL_META_STATE]
+ name = meta_state
+ partitioned = true
+ worker_count = 1
+
+[threadpool.THREAD_POOL_DLOCK]
+ name = dist_lock
+ partitioned = true
+ worker_count = 1
+
+[threadpool.THREAD_POOL_FD]
+ name = fd
+ worker_count = 2
+
+[threadpool.THREAD_POOL_LOCAL_APP]
+ name = local_app
+ worker_count = 2
+
+[threadpool.THREAD_POOL_REPLICATION_LONG]
+ name = rep_long
+ worker_count = 2
+
+[threadpool.THREAD_POOL_BLOCK_SERVICE]
+ name = block_service
+ worker_count = 1
+
+[threadpool.THREAD_POOL_COMPACT]
+ name = compact
+ worker_count = 1
+
+[threadpool.THREAD_POOL_INGESTION]
+ name = ingestion
+ partitioned = false
+ worker_count = 2
+
+[threadpool.THREAD_POOL_SLOG]
+ name = slog
+ worker_count = 1
+
+[threadpool.THREAD_POOL_PLOG]
+ name = plog
+ partitioned = true
+ worker_count = 4
+
+[meta_server]
+ server_list = %{meta.server.list}
+ cluster_root = /pegasus/%{cluster.name}
+ distributed_lock_service_type = distributed_lock_service_zookeeper
+ distributed_lock_service_parameters = /pegasus/%{cluster.name}/lock
+ meta_state_service_type = meta_state_service_zookeeper
+ stable_rs_min_running_seconds = 0
+ server_load_balancer_type = greedy_load_balancer
+ min_live_node_count_for_unfreeze = 1
+ cold_backup_disabled = false
+
+[replication]
+ mutation_2pc_min_replica_count = 1
+ cold_backup_root = %{cluster.name}
+ cluster_name = %{cluster.name}
+
+[meta_server.apps.test]
+ app_name = test
+ app_type = pegasus
+ partition_count = 4
+
+[meta_server.apps.stat]
+ app_name = stat
+ app_type = pegasus
+ partition_count = 4
+
+[pegasus.server]
+ perf_counter_enable_logging = false
+ # Where the metrics are collected. If no value is given, no sink is used.
+ # Options:
+ # - falcon
+ # - prometheus
+ perf_counter_sink =
+ # The HTTP port exposed to Prometheus for pulling metrics from pegasus
server.
+ prometheus_port = @PROMETHEUS_PORT@
+
+[pegasus.collector]
+ available_detect_app = test
+ available_detect_alert_script_dir = ./package/bin
+ usage_stat_app = stat
+
+[pegasus.clusters]
+ %{cluster.name} = %{meta.server.list}
+
+
+# The group of clusters participating in duplication.
+# Each cluster is assigned with a unique cluster id [1, 127] to identify which
cluster
+# the write comes from.
+[duplication-group]
+ onebox = 1
+ onebox2 = 2
+
+[zookeeper]
+ hosts_list = zookeeper:2181
+ timeout_ms = 60000
+ logfile = zoo.log
+
+[task..default]
+ is_trace = false
+ is_profile = false
+ allow_inline = false
+ fast_execution_in_network_thread = false
+ rpc_call_header_format = NET_HDR_DSN
+ rpc_call_channel = RPC_CHANNEL_TCP
+ rpc_timeout_milliseconds = 5000
+
+[task.RPC_PREPARE]
+ is_profile = true
+
+[task.RPC_PREPARE_ACK]
+ is_profile = true
+
+[task.RPC_RRDB_RRDB_PUT]
+ is_profile = true
+ profiler::size.request.server = true
+
+[task.RPC_RRDB_RRDB_PUT_ACK]
+ is_profile = true
+
+[task.RPC_RRDB_RRDB_MULTI_PUT]
+ is_profile = true
+ profiler::size.request.server = true
+
+[task.RPC_RRDB_RRDB_MULTI_PUT_ACK]
+ is_profile = true
+
+[task.RPC_RRDB_RRDB_REMOVE]
+ is_profile = true
+
+[task.RPC_RRDB_RRDB_REMOVE_ACK]
+ is_profile = true
+
+[task.RPC_RRDB_RRDB_MULTI_REMOVE]
+ is_profile = true
+
+[task.RPC_RRDB_RRDB_MULTI_REMOVE_ACK]
+ is_profile = true
+
+[task.RPC_RRDB_RRDB_INCR]
+ is_profile = true
+
+[task.RPC_RRDB_RRDB_INCR_ACK]
+ is_profile = true
+
+[task.RPC_RRDB_RRDB_CHECK_AND_SET]
+ is_profile = true
+
+[task.RPC_RRDB_RRDB_CHECK_AND_SET_ACK]
+ is_profile = true
+
+[task.RPC_RRDB_RRDB_CHECK_AND_MUTATE]
+ is_profile = true
+
+[task.RPC_RRDB_RRDB_CHECK_AND_MUTATE_ACK]
+ is_profile = true
+
+[task.RPC_RRDB_RRDB_GET]
+ is_profile = true
+ profiler::size.response.server = true
+
+[task.RPC_RRDB_RRDB_GET_ACK]
+ is_profile = true
+
+[task.RPC_RRDB_RRDB_MULTI_GET]
+ is_profile = true
+ profiler::size.response.server = true
diff --git a/docker/pegasus-docker-compose/docker-compose.yml
b/docker/pegasus-docker-compose/docker-compose.yml
new file mode 100644
index 00000000..e6a4d5bc
--- /dev/null
+++ b/docker/pegasus-docker-compose/docker-compose.yml
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+version: "3.3"
+
+networks:
+ frontend:
+ ipam:
+ config:
+ - subnet: @[email protected]/24
+services:
+ zookeeper:
+ image: zookeeper:3.4
+ ports:
+ - "2181"
+ restart: on-failure
+ hostname: zookeeper
+ environment:
+ ZOO_MY_ID: 1
+ ZOO_SERVERS: server.1=zookeeper:2888:3888
+ networks:
+ frontend:
diff --git a/docker/pegasus-docker-compose/image_for_prebuilt_bin/Dockerfile
b/docker/pegasus-docker-compose/image_for_prebuilt_bin/Dockerfile
new file mode 100644
index 00000000..07fe9952
--- /dev/null
+++ b/docker/pegasus-docker-compose/image_for_prebuilt_bin/Dockerfile
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM apachepegasus/build-env:ubuntu2004
+
+ARG SERVER_PKG_NAME
+
+COPY ./$SERVER_PKG_NAME.tar.gz /
+RUN cd / \
+ && tar xvf /$SERVER_PKG_NAME.tar.gz \
+ && mv $SERVER_PKG_NAME pegasus \
+ && rm /$SERVER_PKG_NAME.tar.gz
+
+COPY ./entrypoint.sh /
+RUN chmod +x /entrypoint.sh
+
+ENV LD_LIBRARY_PATH=/pegasus/bin
+
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/docker/pegasus-docker-compose/image_for_prebuilt_bin/entrypoint.sh
b/docker/pegasus-docker-compose/image_for_prebuilt_bin/entrypoint.sh
new file mode 100644
index 00000000..a6c71e99
--- /dev/null
+++ b/docker/pegasus-docker-compose/image_for_prebuilt_bin/entrypoint.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#!/bin/bash
+
+if [ -f /pegasus/bin/config_hdfs.sh ]; then
+ source /pegasus/bin/config_hdfs.sh
+fi
+
+/pegasus/bin/pegasus_server /pegasus/bin/config.ini -app_list "$1"
diff --git a/docker/pegasus-docker-compose/prepare.sh
b/docker/pegasus-docker-compose/prepare.sh
new file mode 100755
index 00000000..8aa64b05
--- /dev/null
+++ b/docker/pegasus-docker-compose/prepare.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#!/bin/bash
+
+cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1
+source cluster_args.sh
+
+if ! [[ -x "$(command -v docker-compose)" ]]; then
+ echo 'ERROR: docker-compose is not installed.' >&2
+ echo 'See this document for installation manual:' >&2
+ echo ' https://docs.docker.com/compose/install' >&2
+ exit 1
+fi
+
+if [[ -d "${DOCKER_DIR}" ]]; then
+ echo "ERROR: ${DOCKER_DIR} already exists, please remove it first" >&2
+ exit 1
+fi
+
+mkdir -p "${DOCKER_DIR}"
+
+cp -f "${ROOT}"/config.min.ini "${DOCKER_DIR}/config.ini"
+sed -i "s/%{cluster.name}/${CLUSTER_NAME}/g" "${DOCKER_DIR}/config.ini"
+sed -i "s/allow_non_idempotent_write = false/allow_non_idempotent_write =
${IDEMPOTENT}/" "${DOCKER_DIR}/config.ini"
+for i in $(seq "${META_COUNT}"); do
+ meta_ip=${META_IP_PREFIX}.1$((i))
+ if [ "${i}" -eq 1 ]; then
+ meta_list="${meta_ip}:$META_PORT"
+ else
+ meta_list="$meta_list,${meta_ip}:$META_PORT"
+ fi
+done
+sed -i "s/%{meta.server.list}/$meta_list/g" "${DOCKER_DIR}/config.ini"
+sed -i "s/%{zk.server.list}/${zookeeper_addr}/g" "${DOCKER_DIR}/config.ini"
+sed -i "s/%{meta.port}/$META_PORT/g" "${DOCKER_DIR}/config.ini"
+
+cp -f "${ROOT}"/docker-compose.yml "${DOCKER_DIR}"
+for i in $(seq "${META_COUNT}"); do
+ meta_port=$((META_PORT+i-1))
+ echo " meta$((i)):
+ image: @IMAGE_NAME@
+ ports:
+ - $meta_port:$meta_port
+ volumes:
+ - ./config.ini:/pegasus/bin/config.ini:ro
+ - ./meta$((i))/data:/pegasus/data
+ command:
+ - meta
+ privileged: true
+ networks:
+ frontend:
+ ipv4_address: @[email protected]$((i))
+ restart: on-failure" >> "${DOCKER_DIR}"/docker-compose.yml
+ meta_ip=$(hostname -I | cut -d' ' -f1)
+ echo "META$((i))_ADDRESS=$meta_ip:$meta_port"
+done
+for i in $(seq "${REPLICA_COUNT}"); do
+ echo " replica$((i)):
+ image: @IMAGE_NAME@
+ ports:
+ - 34801
+ volumes:
+ - ./config.ini:/pegasus/bin/config.ini:ro
+ - ./replica$((i))/data:/pegasus/data
+ - ./replica$((i))/slog:/pegasus/slog
+ command:
+ - replica
+ privileged: true
+ restart: on-failure
+ networks:
+ frontend:" >> "${DOCKER_DIR}"/docker-compose.yml
+done
+sed -i "s/@META_IP_PREFIX@/${META_IP_PREFIX}/g"
"${DOCKER_DIR}"/docker-compose.yml
+sed -i "s/@IMAGE_NAME@/${IMAGE_NAME}/g" "${DOCKER_DIR}"/docker-compose.yml
+sed -i "s/@META_PORT@/${META_PORT}/g" "${DOCKER_DIR}"/docker-compose.yml
+
+echo "${DOCKER_DIR} is ready"
diff --git a/docker/pegasus-docker-compose/start_onebox.sh
b/docker/pegasus-docker-compose/start_onebox.sh
new file mode 100755
index 00000000..3075a10b
--- /dev/null
+++ b/docker/pegasus-docker-compose/start_onebox.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#!/usr/bin/env bash
+
+set -e
+
+cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1
+
+source cluster_args.sh
+
+cd "${ROOT}" || exit 1
+
+"${ROOT}"/clear_onebox.sh
+"${ROOT}"/prepare.sh
+
+cd "${DOCKER_DIR}" || exit 1
+pwd
+
+docker-compose up -d
+
+function print_nodes() {
+ cd "${DOCKER_DIR}" || exit 1
+ echo ""
+ docker-compose ps
+ echo ""
+ cd - || exit 1
+}
+
+print_nodes
diff --git a/docker/thirdparties-bin/Dockerfile
b/docker/thirdparties-bin/Dockerfile
new file mode 100644
index 00000000..bcd3866f
--- /dev/null
+++ b/docker/thirdparties-bin/Dockerfile
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG OS_VERSION=centos7
+
+FROM apachepegasus/thirdparties-src as builder
+
+ARG OS_VERSION=centos7
+
+FROM apachepegasus/build-env:${OS_VERSION}
+
+WORKDIR /root
+
+COPY --from=builder /root/thirdparties-src.zip /root/thirdparties-src.zip
+
+# NOTE: the rocksdb lib is built in portable mode.
+RUN git clone --depth=1 https://github.com/XiaoMi/rdsn.git \
+ && cd rdsn/thirdparty \
+ && unzip /root/thirdparties-src.zip -d . \
+ && cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ . \
+ && cmake --build build/ -j $(($(nproc)/2+1)) \
+ && zip -r ~/thirdparties-bin.zip output/ build/Source/rocksdb/cmake
build/Source/http-parser build/Source/hadoop build/Download/zookeeper \
+ && cd ~ \
+ && rm -rf rdsn;
diff --git a/docker/thirdparties-src/Dockerfile
b/docker/thirdparties-src/Dockerfile
new file mode 100644
index 00000000..16de8cf3
--- /dev/null
+++ b/docker/thirdparties-src/Dockerfile
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# use dockerhub by default
+ARG REGISTRY_REPO=apachepegasus
+
+FROM ${REGISTRY_REPO}/build-env:centos7 as builder
+
+WORKDIR /root
+
+ARG TAGNAME=master
+
+RUN git clone --depth=1 --branch=${TAGNAME} https://github.com/XiaoMi/rdsn.git
+
+RUN cd rdsn/thirdparty \
+ && mkdir -p build \
+ && cmake -DCMAKE_BUILD_TYPE=Release -B build/ . \
+ && cmake --build build/ -j $(($(nproc)/2+1))
+
+RUN cd rdsn/thirdparty \
+ && zip -r ~/thirdparties-src.zip build/Download \
+ && cd ~ \
+ && rm -rf rdsn;
+
+# Unzip thirdparties-src.zip to rdsn/thirdparty, the download will
automatically be skipped.
+
+FROM alpine
+
+COPY --from=builder /root/thirdparties-src.zip /root/thirdparties-src.zip
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
