This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new cbc160463 chore(scala-client-ci): build the server instead of
downloading it from external site (#1813)
cbc160463 is described below
commit cbc160463fc1d11eec56ece52e0160d924711e60
Author: Yingchun Lai <[email protected]>
AuthorDate: Tue Dec 26 16:57:24 2023 +0800
chore(scala-client-ci): build the server instead of downloading it from
external site (#1813)
https://github.com/apache/incubator-pegasus/issues/1779
- Separate lint and test jobs
- Build Pegasus server binaries in the CI procedure instead of downloading
it from any site
- Remove scala-client/scripts/ci-test.sh and
scala-client/scripts/format-all.sh
- Fix build failures
---
.github/workflows/lint_and_test_scala-client.yml | 111 +++++++++++++++++++++
.github/workflows/test_scala-client.yml | 55 ----------
scala-client/README.md | 27 ++++-
scala-client/scripts/ci-test.sh | 73 --------------
scala-client/scripts/format-all.sh | 23 -----
.../apache/pegasus/scalaclient/PegasusUtil.scala | 2 +-
.../scalaclient/ScalaPegasusAsyncTable.scala | 5 +-
7 files changed, 142 insertions(+), 154 deletions(-)
diff --git a/.github/workflows/lint_and_test_scala-client.yml
b/.github/workflows/lint_and_test_scala-client.yml
new file mode 100644
index 000000000..b261902e7
--- /dev/null
+++ b/.github/workflows/lint_and_test_scala-client.yml
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Lint and Test - scala client
+on:
+ pull_request:
+ branches:
+ - master
+ - 'v[0-9]+.*' # release branch
+ paths:
+ - .github/workflows/lint_and_test_scala-client.yml
+ - scala-client/**
+
+ workflow_dispatch:
+
+defaults:
+ run:
+ shell: bash
+
+env:
+ ARTIFACT_NAME: release_for_scala_client
+
+jobs:
+ format:
+ name: Format
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/setup-java@v1
+ with:
+ java-version: 8
+ - name: format
+ working-directory: ./scala-client
+ run: sbt scalafmtSbtCheck scalafmtCheck test:scalafmtCheck
+
+ build_server:
+ name: Build server
+ needs: format
+ runs-on: ubuntu-latest
+ env:
+ USE_JEMALLOC: OFF
+ BUILD_OPTIONS: -t release
+ container:
+ image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
+ steps:
+ - uses: actions/checkout@v3
+ - uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - uses: "./.github/actions/build_pegasus"
+ - uses: "./.github/actions/upload_artifact"
+
+ test:
+ name: Test Scala client
+ needs: build_server
+ runs-on: ubuntu-22.04
+ container:
+ image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
+ strategy:
+ fail-fast: false
+ matrix:
+ java: [ '8', '11']
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/cache@v2
+ with:
+ path: ~/.m2/repository
+ key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-maven-
+ - uses: actions/setup-java@v1
+ with:
+ java-version: ${{ matrix.java }}
+ - name: Setup sbt
+ run: |
+ curl -s "https://get.sdkman.io"; | bash
+ source /github/home/.sdkman/bin/sdkman-init.sh
+ sdk install sbt
+ sbt -V
+ - name: Recompile thrift
+ working-directory: ./java-client/scripts
+ run: ./recompile_thrift.sh
+ - name: Build Java client
+ working-directory: ./java-client
+ run: |
+ mvn clean package -DskipTests -Dcheckstyle.skip=true
+ mvn clean install -DskipTests -Dcheckstyle.skip=true
+ - name: Download artifact
+ uses: "./.github/actions/download_artifact"
+ - name: Start Pegasus cluster
+ run: |
+ export
LD_LIBRARY_PATH=$(pwd)/thirdparty/output/lib:${JAVA_HOME}/jre/lib/amd64/server
+ ulimit -s unlimited
+ ./run.sh start_onebox
+ - name: Run Scala client tests
+ working-directory: ./scala-client
+ run: |
+ source /github/home/.sdkman/bin/sdkman-init.sh
+ sbt test
diff --git a/.github/workflows/test_scala-client.yml
b/.github/workflows/test_scala-client.yml
deleted file mode 100644
index 047e55042..000000000
--- a/.github/workflows/test_scala-client.yml
+++ /dev/null
@@ -1,55 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Test - scala client
-on:
- pull_request:
- branches:
- - master
- - 'v[0-9]+.*' # release branch
- paths:
- - .github/workflows/test_scala-client.yml
- - scala-client/**
-
- workflow_dispatch:
-
-defaults:
- run:
- shell: bash
-
-jobs:
- test:
- name: test
- runs-on: ubuntu-latest
- strategy:
- matrix:
- java: [ '8', '11']
- steps:
- - uses: actions/cache@v2
- with:
- path: ~/.m2/repository
- key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-maven-
- - uses: actions/checkout@v3
- - uses: actions/setup-java@v1
- with:
- java-version: ${{ matrix.java }}
- - name: ci
- working-directory: scala-client
- run: ./scripts/ci-test.sh
diff --git a/scala-client/README.md b/scala-client/README.md
index 5d86821b5..7aa454f0d 100644
--- a/scala-client/README.md
+++ b/scala-client/README.md
@@ -23,7 +23,6 @@ It's built on top of the java client
[apache/incubator-pegasus/java-client](http
## Features:
* Scala friendly.
-
* Serialize/deserialize automatically.
## Example:
@@ -53,3 +52,29 @@ It's built on top of the java client
[apache/incubator-pegasus/java-client](http
c.close
```
+
+## Development
+
+### Format the code
+
+Use scala format tool, see https://github.com/scalameta/scalafmt
+```
+sbt scalafmtSbt scalafmt test:scalafmt
+```
+
+### Run tests
+
+NOTE: It requires the Pegasus
[onebox](https://pegasus.apache.org/overview/onebox/) has been started.
+
+Build Java dependency at first, then build and test Scala client.
+```
+cd ${PROJECT_ROOT}/java-client/scripts
+./recompile_thrift.sh
+
+cd ${PROJECT_ROOT}/java-client
+mvn clean package -DskipTests -Dcheckstyle.skip=true
+mvn clean install -DskipTests -Dcheckstyle.skip=true
+
+cd ${PROJECT_ROOT}/scala-client
+sbt test
+```
diff --git a/scala-client/scripts/ci-test.sh b/scala-client/scripts/ci-test.sh
deleted file mode 100755
index dddb168ef..000000000
--- a/scala-client/scripts/ci-test.sh
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-set -e
-
-SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
-PROJECT_DIR=$(dirname "${SCRIPT_DIR}")
-cd "${PROJECT_DIR}" || exit 1
-
-# lint all scripts, abort if there's any warning.
-function shellcheck_must_pass()
-{
- if [[ $(shellcheck "$1") ]]; then
- echo "shellcheck $1 failed"
- shellcheck "$1"
- exit 1
- fi
-}
-shellcheck_must_pass ./scripts/ci-test.sh
-
-# check format
-sbt scalafmtSbtCheck scalafmtCheck test:scalafmtCheck
-
-# install java-client dependency
-git clone https://github.com/apache/incubator-pegasus.git
-cd incubator-pegasus/java-client
-git checkout master
-cd scripts
-echo "run recompile_thrift.sh"
-./recompile_thrift.sh
-cd ..
-mvn clean package -DskipTests -Dcheckstyle.skip=true
-mvn clean install -DskipTests -Dcheckstyle.skip=true
-cd ..
-
-# start pegasus onebox environment
-PEGASUS_PKG="pegasus-tools-2.0.0-5d969e8-glibc2.12-release"
-PEGASUS_PKG_URL="https://github.com/apache/incubator-pegasus/releases/download/v2.0.0/pegasus-tools-2.0.0-5d969e8-glibc2.12-release.tar.gz";
-
-# start pegasus onebox environment
-if [ ! -f $PEGASUS_PKG.tar.gz ]; then
- wget $PEGASUS_PKG_URL
- tar xvf $PEGASUS_PKG.tar.gz
-fi
-cd $PEGASUS_PKG
-
-./run.sh start_onebox -w
-cd ../
-
-if ! sbt test
-then
- cd $PEGASUS_PKG
- ./run.sh list_onebox
- exit 1
-fi
diff --git a/scala-client/scripts/format-all.sh
b/scala-client/scripts/format-all.sh
deleted file mode 100755
index 3ff058187..000000000
--- a/scala-client/scripts/format-all.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# scala format tool,see https://github.com/scalameta/scalafmt
-sbt scalafmtSbt scalafmt test:scalafmt
diff --git
a/scala-client/src/main/scala/org/apache/pegasus/scalaclient/PegasusUtil.scala
b/scala-client/src/main/scala/org/apache/pegasus/scalaclient/PegasusUtil.scala
index 7fac3d550..8240e8687 100644
---
a/scala-client/src/main/scala/org/apache/pegasus/scalaclient/PegasusUtil.scala
+++
b/scala-client/src/main/scala/org/apache/pegasus/scalaclient/PegasusUtil.scala
@@ -131,7 +131,7 @@ private[scalaclient] trait PegasusUtil {
def convertMultiGetResult[S](result: PegasusTableInterface.MultiGetResult)(
implicit sSer: SER[S]) = {
- MultiGetResult(result.allFetched, result.values.toList.map { p =>
+ MultiGetResult(result.isAllFetched(), result.getValues().toList.map { p =>
(sSer.deserialize(p.getLeft), p.getRight)
})
}
diff --git
a/scala-client/src/main/scala/org/apache/pegasus/scalaclient/ScalaPegasusAsyncTable.scala
b/scala-client/src/main/scala/org/apache/pegasus/scalaclient/ScalaPegasusAsyncTable.scala
index 90534379d..969d9365d 100644
---
a/scala-client/src/main/scala/org/apache/pegasus/scalaclient/ScalaPegasusAsyncTable.scala
+++
b/scala-client/src/main/scala/org/apache/pegasus/scalaclient/ScalaPegasusAsyncTable.scala
@@ -21,7 +21,10 @@ package org.apache.pegasus.scalaclient
import org.apache.pegasus.client.{PException, PegasusTableInterface => ITable}
import org.apache.pegasus.scalaclient.{Serializer => SER}
-import io.netty.util.concurrent.{GenericFutureListener, Future => NFuture}
+import org.apache.pegasus.thirdparty.io.netty.util.concurrent.{
+ GenericFutureListener,
+ Future => NFuture
+}
import scala.collection.JavaConversions._
import scala.concurrent.duration._
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
