This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 1454aaafe [VL][CI] Cache native libraries to re-use them in Spark test
jobs (#5768)
1454aaafe is described below
commit 1454aaafe4202ceff5010bf7376ac09d67de31ce
Author: PHILO-HE <[email protected]>
AuthorDate: Fri May 17 20:41:20 2024 +0800
[VL][CI] Cache native libraries to re-use them in Spark test jobs (#5768)
Co-authored-by: Yuan Zhou <[email protected]>
---
.github/workflows/velox_docker.yml | 291 ++++++++++++++++++++++---------------
1 file changed, 174 insertions(+), 117 deletions(-)
diff --git a/.github/workflows/velox_docker.yml
b/.github/workflows/velox_docker.yml
index a92abda3c..b7ee3be41 100644
--- a/.github/workflows/velox_docker.yml
+++ b/.github/workflows/velox_docker.yml
@@ -47,7 +47,7 @@ concurrency:
cancel-in-progress: true
jobs:
- build-native-lib:
+ build-native-lib-centos-7:
runs-on: ubuntu-20.04
container: apache/gluten:gluten-vcpkg-builder_2024_03_17 # centos7 with
dependencies installed
steps:
@@ -68,10 +68,10 @@ jobs:
- uses: actions/upload-artifact@v2
with:
path: ./cpp/build/releases/
- name: velox-native-lib-${{github.sha}}
+ name: velox-native-lib-centos-7-${{github.sha}}
run-tpc-test-ubuntu:
- needs: build-native-lib
+ needs: build-native-lib-centos-7
strategy:
fail-fast: false
matrix:
@@ -95,7 +95,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v2
with:
- name: velox-native-lib-${{github.sha}}
+ name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/releases
- name: Setup java and maven
run: |
@@ -119,7 +119,7 @@ jobs:
--local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1
run-tpc-test-centos:
- needs: build-native-lib
+ needs: build-native-lib-centos-7
strategy:
fail-fast: false
matrix:
@@ -143,7 +143,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v2
with:
- name: velox-native-lib-${{github.sha}}
+ name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/releases
- name: Update mirror list
if: matrix.os == 'centos:8'
@@ -195,7 +195,7 @@ jobs:
--extra-conf=spark.gluten.sql.ras.enabled=true
run-tpc-test-ubuntu-oom:
- needs: build-native-lib
+ needs: build-native-lib-centos-7
strategy:
fail-fast: false
matrix:
@@ -218,7 +218,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v2
with:
- name: velox-native-lib-${{github.sha}}
+ name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/releases
- name: Setup java and maven
run: |
@@ -292,7 +292,7 @@ jobs:
-d=OFFHEAP_SIZE:1g,spark.memory.offHeap.size=1g || true
run-tpc-test-ubuntu-randomkill:
- needs: build-native-lib
+ needs: build-native-lib-centos-7
strategy:
fail-fast: false
matrix:
@@ -315,7 +315,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v2
with:
- name: velox-native-lib-${{github.sha}}
+ name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/releases
- name: Setup java and maven
run: |
@@ -339,7 +339,7 @@ jobs:
--skip-data-gen --random-kill-tasks
run-tpc-test-ubuntu-sf30:
- needs: build-native-lib
+ needs: build-native-lib-centos-7
strategy:
fail-fast: false
matrix:
@@ -363,7 +363,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v2
with:
- name: velox-native-lib-${{github.sha}}
+ name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/releases
- name: Setup java and maven
run: |
@@ -391,7 +391,7 @@ jobs:
--skip-data-gen --shard=${{ matrix.shard }}
run-tpc-test-centos8-uniffle:
- needs: build-native-lib
+ needs: build-native-lib-centos-7
strategy:
fail-fast: false
matrix:
@@ -403,7 +403,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v2
with:
- name: velox-native-lib-${{github.sha}}
+ name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/releases
- name: Update mirror list
run: |
@@ -451,7 +451,7 @@ jobs:
--local --preset=velox-with-uniffle --benchmark-type=h
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1
run-tpc-test-ubuntu-2204-celeborn:
- needs: build-native-lib
+ needs: build-native-lib-centos-7
strategy:
fail-fast: false
matrix:
@@ -464,7 +464,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v2
with:
- name: velox-native-lib-${{github.sha}}
+ name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/releases
- name: Setup java and maven
run: |
@@ -494,13 +494,74 @@ jobs:
GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
--local --preset=velox-with-celeborn --benchmark-type=ds
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=8 --iterations=1
+ build-native-lib-centos-8:
+ runs-on: ubuntu-20.04
+ container: ghcr.io/facebookincubator/velox-dev:circleci-avx
+ steps:
+ - uses: actions/checkout@v2
+ - name: Generate cache key
+ run: |
+ echo ${{ hashFiles('./ep/build-velox/src/**', './dev/**', './cpp/*',
'./github/workflows/*') }} > cache-key
+ - name: Cache
+ id: cache
+ uses: actions/cache/restore@v3
+ with:
+ path: ./cpp/build/releases/
+ key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}
+ - name: Setup build dependency
+ if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+ run: |
+ yum install sudo patch java-1.8.0-openjdk-devel wget -y
+ # Required by building arrow java.
+ wget
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+ tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8
/usr/lib/maven
+ echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
+ - name: Build Gluten Velox third party
+ if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+ run: |
+ cd ep/build-velox/src
+ ./get_velox.sh
+ source /opt/rh/gcc-toolset-9/enable
+ ./build_arrow_deps_centos8.sh
+ ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF
--build_test_utils=ON
+ cd $GITHUB_WORKSPACE/cpp
+ ./compile.sh --build_velox_backend=ON --build_protobuf=ON
--build_tests=ON --build_examples=ON --build_benchmarks=ON
+ - uses: actions/upload-artifact@v2
+ with:
+ name: velox-native-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/releases/
+ - uses: actions/upload-artifact@v2
+ with:
+ name: udf-example-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/velox/udf/examples/
+ - uses: actions/upload-artifact@v2
+ with:
+ name: arrow-jars-centos-8-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
+
run-spark-test-spark32:
+ needs: build-native-lib-centos-8
runs-on: ubuntu-20.04
container: ghcr.io/facebookincubator/velox-dev:circleci-avx
env:
CCACHE_DIR: "${{ github.workspace }}/.ccache"
steps:
- uses: actions/checkout@v2
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v2
+ with:
+ name: velox-native-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Download UDF Example Lib
+ uses: actions/download-artifact@v2
+ with:
+ name: udf-example-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/velox/udf/examples/
+ - name: Download Arrow Jars
+ uses: actions/download-artifact@v2
+ with:
+ name: arrow-jars-centos-8-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
@@ -517,18 +578,6 @@ jobs:
working-directory: ${{ github.workspace }}
run: |
mkdir -p '${{ env.CCACHE_DIR }}'
- - name: Build Gluten velox third party
- run: |
- cd ep/build-velox/src && \
- ./get_velox.sh && \
- source /opt/rh/gcc-toolset-9/enable && \
- ./build_arrow_deps_centos8.sh && \
- ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF
--build_test_utils=ON
- - name: Build Gluten CPP library
- run: |
- cd $GITHUB_WORKSPACE/cpp && \
- source /opt/rh/gcc-toolset-9/enable && \
- ./compile.sh --build_velox_backend=ON --build_protobuf=ON
--build_tests=ON --build_examples=ON --build_benchmarks=ON
- name: Gluten CPP Test
run: |
cd $GITHUB_WORKSPACE/cpp/build && \
@@ -562,19 +611,30 @@ jobs:
with:
name: golden-files-spark32
path: /tmp/tpch-approved-plan/**
- - name: Gluten CPP Benchmark Test
- run: |
- # This test depends on example.json generated by the above mvn test.
- cd $GITHUB_WORKSPACE/cpp/build/velox/benchmarks && \
- ./generic_benchmark --run-example --with-shuffle --threads 1
--iterations 1
+ # - name: Gluten CPP Benchmark Test
+ # run: |
+ # # This test depends on example.json generated by the above mvn
test.
+ # cd $GITHUB_WORKSPACE/cpp/build/velox/benchmarks && \
+ # ./generic_benchmark --run-example --with-shuffle --threads 1
--iterations 1
run-spark-test-spark32-slow:
+ needs: build-native-lib-centos-8
runs-on: ubuntu-20.04
container: ghcr.io/facebookincubator/velox-dev:circleci-avx
env:
CCACHE_DIR: "${{ github.workspace }}/.ccache"
steps:
- uses: actions/checkout@v2
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v2
+ with:
+ name: velox-native-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Download Arrow Jars
+ uses: actions/download-artifact@v2
+ with:
+ name: arrow-jars-centos-8-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
@@ -591,18 +651,6 @@ jobs:
working-directory: ${{ github.workspace }}
run: |
mkdir -p '${{ env.CCACHE_DIR }}'
- - name: Build Gluten velox third party
- run: |
- cd ep/build-velox/src && \
- ./get_velox.sh && \
- source /opt/rh/gcc-toolset-9/enable && \
- ./build_arrow_deps_centos8.sh && \
- ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF
--build_test_utils=ON
- - name: Build Gluten CPP library
- run: |
- cd $GITHUB_WORKSPACE/cpp && \
- source /opt/rh/gcc-toolset-9/enable && \
- ./compile.sh --build_velox_backend=ON --build_protobuf=ON
- name: Prepare spark.test.home for Spark 3.2.2 (slow tests)
run: |
cd $GITHUB_WORKSPACE// && \
@@ -616,12 +664,28 @@ jobs:
mvn -ntp clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox
-Pceleborn -Piceberg -Pdelta
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/"
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
run-spark-test-spark33:
+ needs: build-native-lib-centos-8
runs-on: ubuntu-20.04
container: ghcr.io/facebookincubator/velox-dev:circleci-avx
env:
CCACHE_DIR: "${{ github.workspace }}/.ccache"
steps:
- uses: actions/checkout@v2
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v2
+ with:
+ name: velox-native-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Download UDF Example Lib
+ uses: actions/download-artifact@v2
+ with:
+ name: udf-example-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/velox/udf/examples/
+ - name: Download Arrow Jars
+ uses: actions/download-artifact@v2
+ with:
+ name: arrow-jars-centos-8-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
@@ -638,18 +702,6 @@ jobs:
working-directory: ${{ github.workspace }}
run: |
mkdir -p '${{ env.CCACHE_DIR }}'
- - name: Build Gluten velox third party
- run: |
- cd ep/build-velox/src && \
- ./get_velox.sh && \
- source /opt/rh/gcc-toolset-9/enable && \
- ./build_arrow_deps_centos8.sh && \
- ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF
--build_test_utils=ON
- - name: Build Gluten CPP library
- run: |
- cd $GITHUB_WORKSPACE/cpp && \
- source /opt/rh/gcc-toolset-9/enable && \
- ./compile.sh --build_velox_backend=ON --build_protobuf=ON
--build_tests=ON --build_examples=ON --build_benchmarks=ON
- name: Prepare spark.test.home for Spark 3.3.1 (other tests)
run: |
cd $GITHUB_WORKSPACE/ && \
@@ -682,12 +734,23 @@ jobs:
run-spark-test-spark33-slow:
+ needs: build-native-lib-centos-8
runs-on: ubuntu-20.04
container: ghcr.io/facebookincubator/velox-dev:circleci-avx
env:
CCACHE_DIR: "${{ github.workspace }}/.ccache"
steps:
- uses: actions/checkout@v2
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v2
+ with:
+ name: velox-native-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Download Arrow Jars
+ uses: actions/download-artifact@v2
+ with:
+ name: arrow-jars-centos-8-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
@@ -704,18 +767,6 @@ jobs:
working-directory: ${{ github.workspace }}
run: |
mkdir -p '${{ env.CCACHE_DIR }}'
- - name: Build Gluten velox third party
- run: |
- cd ep/build-velox/src && \
- ./get_velox.sh && \
- source /opt/rh/gcc-toolset-9/enable && \
- ./build_arrow_deps_centos8.sh && \
- ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF
--build_test_utils=ON
- - name: Build Gluten CPP library
- run: |
- cd $GITHUB_WORKSPACE/cpp && \
- source /opt/rh/gcc-toolset-9/enable && \
- ./compile.sh --build_velox_backend=ON --build_protobuf=ON
- name: Prepare spark.test.home for Spark 3.3.1 (slow tests)
run: |
cd $GITHUB_WORKSPACE// && \
@@ -729,12 +780,28 @@ jobs:
mvn -ntp clean install -Pspark-3.3 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Pspark-ut
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/"
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
run-spark-test-spark34:
+ needs: build-native-lib-centos-8
runs-on: ubuntu-20.04
container: ghcr.io/facebookincubator/velox-dev:circleci-avx
env:
CCACHE_DIR: "${{ github.workspace }}/.ccache"
steps:
- uses: actions/checkout@v2
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v2
+ with:
+ name: velox-native-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Download UDF Example Lib
+ uses: actions/download-artifact@v2
+ with:
+ name: udf-example-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/velox/udf/examples/
+ - name: Download Arrow Jars
+ uses: actions/download-artifact@v2
+ with:
+ name: arrow-jars-centos-8-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
@@ -751,18 +818,6 @@ jobs:
working-directory: ${{ github.workspace }}
run: |
mkdir -p '${{ env.CCACHE_DIR }}'
- - name: Build Gluten velox third party
- run: |
- cd ep/build-velox/src && \
- ./get_velox.sh && \
- source /opt/rh/gcc-toolset-9/enable && \
- ./build_arrow_deps_centos8.sh && \
- ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF
--build_test_utils=ON
- - name: Build Gluten CPP library
- run: |
- cd $GITHUB_WORKSPACE/cpp && \
- source /opt/rh/gcc-toolset-9/enable && \
- ./compile.sh --build_velox_backend=ON --build_protobuf=ON
--build_tests=ON --build_examples=ON --build_benchmarks=ON
- name: Prepare spark.test.home for Spark 3.4.2 (other tests)
run: |
cd $GITHUB_WORKSPACE/ && \
@@ -795,12 +850,23 @@ jobs:
run-spark-test-spark34-slow:
+ needs: build-native-lib-centos-8
runs-on: ubuntu-20.04
container: ghcr.io/facebookincubator/velox-dev:circleci-avx
env:
CCACHE_DIR: "${{ github.workspace }}/.ccache"
steps:
- uses: actions/checkout@v2
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v2
+ with:
+ name: velox-native-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Download Arrow Jars
+ uses: actions/download-artifact@v2
+ with:
+ name: arrow-jars-centos-8-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
@@ -817,18 +883,6 @@ jobs:
working-directory: ${{ github.workspace }}
run: |
mkdir -p '${{ env.CCACHE_DIR }}'
- - name: Build Gluten velox third party
- run: |
- cd ep/build-velox/src && \
- ./get_velox.sh && \
- source /opt/rh/gcc-toolset-9/enable && \
- ./build_arrow_deps_centos8.sh && \
- ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF
--build_test_utils=ON
- - name: Build Gluten CPP library
- run: |
- cd $GITHUB_WORKSPACE/cpp && \
- source /opt/rh/gcc-toolset-9/enable && \
- ./compile.sh --build_velox_backend=ON --build_protobuf=ON
- name: Prepare spark.test.home for Spark 3.4.2 (slow tests)
run: |
cd $GITHUB_WORKSPACE// && \
@@ -842,12 +896,28 @@ jobs:
mvn -ntp clean install -Pspark-3.4 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Pspark-ut
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/"
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
run-spark-test-spark35:
+ needs: build-native-lib-centos-8
runs-on: ubuntu-20.04
container: ghcr.io/facebookincubator/velox-dev:circleci-avx
env:
CCACHE_DIR: "${{ github.workspace }}/.ccache"
steps:
- uses: actions/checkout@v2
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v2
+ with:
+ name: velox-native-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Download UDF Example Lib
+ uses: actions/download-artifact@v2
+ with:
+ name: udf-example-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/velox/udf/examples/
+ - name: Download Arrow Jars
+ uses: actions/download-artifact@v2
+ with:
+ name: arrow-jars-centos-8-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
@@ -864,18 +934,6 @@ jobs:
working-directory: ${{ github.workspace }}
run: |
mkdir -p '${{ env.CCACHE_DIR }}'
- - name: Build Gluten velox third party
- run: |
- cd ep/build-velox/src && \
- ./get_velox.sh && \
- source /opt/rh/gcc-toolset-9/enable && \
- ./build_arrow_deps_centos8.sh && \
- ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF
--build_test_utils=ON
- - name: Build Gluten CPP library
- run: |
- cd $GITHUB_WORKSPACE/cpp && \
- source /opt/rh/gcc-toolset-9/enable && \
- ./compile.sh --build_velox_backend=ON --build_protobuf=ON
--build_tests=ON --build_examples=ON --build_benchmarks=ON
- name: Prepare spark.test.home for Spark 3.5.1 (other tests)
run: |
cd $GITHUB_WORKSPACE/ && \
@@ -907,12 +965,23 @@ jobs:
path: /tmp/tpch-approved-plan/**
run-spark-test-spark35-slow:
+ needs: build-native-lib-centos-8
runs-on: ubuntu-20.04
container: ghcr.io/facebookincubator/velox-dev:circleci-avx
env:
CCACHE_DIR: "${{ github.workspace }}/.ccache"
steps:
- uses: actions/checkout@v2
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v2
+ with:
+ name: velox-native-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Download Arrow Jars
+ uses: actions/download-artifact@v2
+ with:
+ name: arrow-jars-centos-8-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
- name: Setup build dependency
run: |
yum install sudo patch java-1.8.0-openjdk-devel wget -y
@@ -929,18 +998,6 @@ jobs:
working-directory: ${{ github.workspace }}
run: |
mkdir -p '${{ env.CCACHE_DIR }}'
- - name: Build Gluten velox third party
- run: |
- cd ep/build-velox/src && \
- ./get_velox.sh && \
- source /opt/rh/gcc-toolset-9/enable && \
- ./build_arrow_deps_centos8.sh && \
- ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF
--build_test_utils=ON
- - name: Build Gluten CPP library
- run: |
- cd $GITHUB_WORKSPACE/cpp && \
- source /opt/rh/gcc-toolset-9/enable && \
- ./compile.sh --build_velox_backend=ON --build_protobuf=ON
--build_tests=ON --build_examples=ON --build_benchmarks=ON
- name: Prepare spark.test.home for Spark 3.5.1 (other tests)
run: |
cd $GITHUB_WORKSPACE/ && \
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
