This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 38c9b85 [HUDI-2280] Use GitHub Actions to build different scala spark
versions (#3556)
38c9b85 is described below
commit 38c9b85aa8aa8f72876b112e376f300076fafacd
Author: Raymond Xu <[email protected]>
AuthorDate: Wed Sep 1 08:51:00 2021 -0700
[HUDI-2280] Use GitHub Actions to build different scala spark versions
(#3556)
---
.github/actions/bot/package.json | 36 --------------
.github/actions/bot/src/action.js | 100 --------------------------------------
.github/workflows/bot.yml | 64 +++++++++++-------------
.travis.yml | 50 -------------------
pom.xml | 5 +-
scripts/report_coverage.sh | 34 -------------
scripts/run_travis_tests.sh | 42 ----------------
7 files changed, 32 insertions(+), 299 deletions(-)
diff --git a/.github/actions/bot/package.json b/.github/actions/bot/package.json
deleted file mode 100644
index 0d6e9d1..0000000
--- a/.github/actions/bot/package.json
+++ /dev/null
@@ -1,36 +0,0 @@
-{
- "name": "github_action_ci_bot",
- "version": "1.1.1",
- "description": "CI Bot for GitHub Actions",
- "main": "dist/index.js",
- "scripts": {
- "lint": "eslint 'src/**.js' 'tests/**.js' --fix",
- "test": "eslint 'src/**.js' 'tests/**.js' && jest --coverage",
- "build": "ncc build src/action.js"
- },
- "author": "lamber-ken",
- "license": "Apache LICENSE 2.0",
- "homepage": "https://github.com/apache/hudi";,
- "bugs": {
- "url": "https://github.com/apache/hudi/issues";
- },
- "dependencies": {
- "@actions/core": "^1.2.4",
- "@actions/github": "^2.2.0",
- "@actions/io": "^1.0.2"
- },
- "devDependencies": {
- "@types/jest": "^25.1.4",
- "@typescript-eslint/eslint-plugin": "^2.33.0",
- "@typescript-eslint/parser": "^2.33.0",
- "@zeit/ncc": "^0.22.0",
- "eslint": "^7.0.0",
- "eslint-config-prettier": "^6.11.0",
- "husky": "^4.2.5",
- "jest": "^25.1.0",
- "npm-run-all": "^4.1.5",
- "prettier": "^2.0.5",
- "ts-jest": "^25.2.1",
- "typescript": "^3.8.3"
- }
-}
diff --git a/.github/actions/bot/src/action.js
b/.github/actions/bot/src/action.js
deleted file mode 100644
index cb4f691..0000000
--- a/.github/actions/bot/src/action.js
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-async function check(core, context, github) {
-
- try {
- const provider = process.env.PROVIDER;
- const repository = process.env.REPOSITORY;
- const command = context.payload.comment.body;
-
- if (command !== 'rerun tests') {
- console.log("Invalid command:" + command);
- return;
- }
-
- const {
- data: {
- head: {
- sha: ref,
- }
- }
- } = await github.pulls.get({
- owner: provider,
- repo: repository,
- pull_number: context.issue.number,
- });
-
- const checks = await github.checks.listForRef({
- owner: provider,
- repo: repository,
- ref: ref
- });
-
- checks.data.check_runs.forEach(run => {
-
- if (run.app.owner.login === 'travis-ci') {
- console.log("rerun travis ci check: " + run.external_id);
- rebuild(run.external_id)
- } else {
- console.log("ignore github action check: " + run.id);
- }
-
- });
-
- } catch (e) {
- console.log("check bot run failed: " + e);
- }
-
-}
-
-function rebuild(buildId) {
- const https = require('https');
- const token = process.env.HUDI_TRAVIS_ORG_TOKEN
-
- const options = {
- hostname: 'api.travis-ci.org',
- port: 443,
- path: `/build/${buildId}/restart`,
- method: 'POST',
- headers: {
- 'Content-Type': 'application/json',
- 'Travis-API-Version': 3,
- 'Authorization': `token ${token}`,
- }
- };
-
- const req = https.request(options, function (res) {
- res.setEncoding('utf8');
- res.on('data', function (data) {
- console.log('data: ' + data);
- });
- res.on('error', function (error) {
- console.log('error: ' + error);
- });
- });
- req.on('error', function (e) {
- console.log('problem with request: ' + e.message);
- });
-
- req.end();
-}
-
-module.exports = ({core}, {context}, {github}) => {
- return check(core, context, github);
-}
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index ca34cc3..3cc43fe 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -1,43 +1,35 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI BOT
+name: Java CI
on:
- issue_comment:
- types: [created]
+ push:
+ branches:
+ - master
+ - 'release-*'
+ pull_request:
+ branches:
+ - master
+ - 'release-*'
jobs:
- bot:
+ build:
runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ include:
+ - scala: "scala-2.11"
+ spark: "spark2"
+ - scala: "scala-2.12"
+ spark: "spark3"
steps:
- - name: clone repository
- uses: actions/checkout@v2
-
- - name: bot actions
- uses: actions/github-script@v1
- env:
- PROVIDER : 'apache'
- REPOSITORY: 'hudi'
- HUDI_TRAVIS_ORG_TOKEN: ${{ secrets.HUDI_TRAVIS_ORG_TOKEN }}
+ - uses: actions/checkout@v2
+ - name: Set up JDK 8
+ uses: actions/setup-java@v2
with:
- script: |
- const path = require('path')
- const scriptPath =
path.resolve('.github/actions/bot/src/action.js')
- require(scriptPath)({core}, {context}, {github})
\ No newline at end of file
+ java-version: '8'
+ distribution: 'adopt'
+ architecture: x64
+ - name: Build Project
+ env:
+ SCALA_PROFILE: ${{ matrix.scala }}
+ SPAKR_PROFILE: ${{ matrix.spark }}
+ run: mvn install -P "$SCALA_PROFILE,$SPAKR_PROFILE" -DskipTests=true
-Dmaven.javadoc.skip=true -B -V
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 532099c..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-os: linux
-dist: trusty
-language: java
-jdk:
- - openjdk8
-jobs:
- include:
- - name: "Unit tests for hudi-spark-client"
- env: MODE=unit MODULES=hudi-client/hudi-spark-client
HUDI_QUIETER_LOGGING=1
- - name: "Unit tests for hudi-utilities"
- env: MODE=unit MODULES=hudi-utilities HUDI_QUIETER_LOGGING=1
- - name: "All other unit tests"
- env: MODE=unit MODULES='!hudi-utilities,!hudi-client/hudi-spark-client'
HUDI_QUIETER_LOGGING=1
- - name: "Functional tests"
- env: MODE=functional HUDI_QUIETER_LOGGING=1
- - name: "Integration tests"
- env: MODE=integration HUDI_QUIETER_LOGGING=1
-install: true
-services:
- - docker
-cache:
- directories:
- - "$HOME/.m2"
-notifications:
- slack:
- rooms:
- - secure:
WNIZPBY//xf/xTJL1YUPzvPUDwjawaMM4IJ6IqxjRGcZCmuhNVu2XTJ3aL1g6X7ZcJKxJuwoU/TbSO8Dl6rgWSo/2OfyzBd4ks+hgeCsdycccTcvO8giQO1DOUGUSRdvUzOvKjWVK7iARYzQhoZawAYwI09UJLlwhYRCJ1IKc1ZksrEt964GeEmPyJbwMoZOJVUU84jJIAZPIpOFGTKM652FMermg9yaY2W5oSjDXaV98z0/mJV4Ry++J2v0fvoDs5HxkXYhZJP+dpWR82KDr6Q6LGL5/IlJ+b+IH3pF8LyKR4nCH6l1EZ8KpoFZapyYWYQpXMfQoF2K/JEQkpz1EqBCeEDSJ2+j1PPLhOWXd7ok4DsS26S8BP2ImvyXwua51THN1/r1fCGSIdxiQ5C8aeYmPCSr+oLChCVivEG2eeU34Z1nQJ5aDymNGeFE9qUUpjS0ETfFcjI/WQaA+FiYiPkDfeAo
[...]
- on_pull_requests: false
-script:
- # ping stdout every 9 minutes or Travis kills build
- #
https://docs.travis-ci.com/user/common-build-problems/#Build-times-out-because-no-output-was-received
- - while sleep 9m; do echo "=====[ $SECONDS seconds still running ]=====";
done &
- - scripts/run_travis_tests.sh $MODE $MODULES
-after_success:
- - scripts/report_coverage.sh
diff --git a/pom.xml b/pom.xml
index 0fd756e..65e391a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1418,7 +1418,6 @@
<profile>
<id>scala-2.11</id>
</profile>
-
<profile>
<id>scala-2.12</id>
<properties>
@@ -1460,6 +1459,10 @@
</build>
</profile>
+ <!-- Exists for backwards compatibility; profile doesn't do anything -->
+ <profile>
+ <id>spark2</id>
+ </profile>
<profile>
<id>spark3</id>
<properties>
diff --git a/scripts/report_coverage.sh b/scripts/report_coverage.sh
deleted file mode 100755
index 79bbce0..0000000
--- a/scripts/report_coverage.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-modules=(
- hudi-cli
- hudi-client
- hudi-common
- hudi-examples
- hudi-flink
- hudi-hadoop-mr
- hudi-integ-test
- hudi-spark-datasource
- hudi-sync
- hudi-timeline-service
- hudi-utilities
-)
-for module in "${modules[@]}"; do
- bash <(curl -s https://codecov.io/bash) -s "$module" -F "${module//-/}"
-done
diff --git a/scripts/run_travis_tests.sh b/scripts/run_travis_tests.sh
deleted file mode 100755
index 63fb959..0000000
--- a/scripts/run_travis_tests.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-mode=$1
-modules=$2
-sparkVersion=2.4.4
-hadoopVersion=2.7
-
-if [ "$mode" = "unit" ]; then
- mvn clean install -DskipTests -q
- echo "Running Unit Tests"
- mvn test -Punit-tests -pl "$modules" -B
-elif [ "$mode" = "functional" ]; then
- echo "Running Functional Tests"
- mvn test -Pfunctional-tests -B
-elif [ "$mode" = "integration" ]; then
- echo "Downloading Apache Spark-${sparkVersion}-bin-hadoop${hadoopVersion}"
- wget
http://archive.apache.org/dist/spark/spark-${sparkVersion}/spark-${sparkVersion}-bin-hadoop${hadoopVersion}.tgz
-O /tmp/spark-${sparkVersion}.tgz
- tar -xvf /tmp/spark-${sparkVersion}.tgz
- export SPARK_HOME=$PWD/spark-${sparkVersion}-bin-hadoop${hadoopVersion}
- mkdir /tmp/spark-events/
- echo "Running Integration Tests"
- mvn verify -Pintegration-tests -B
-else
- echo "Unknown mode $mode"
- exit 1
-fi
