This is an automated email from the ASF dual-hosted git repository. kinghao pushed a commit to branch feature-compile-optimization in repository https://gitbox.apache.org/repos/asf/linkis.git
commit 130b744a745ee4ad9fe2626b6e4322777f1a639b Author: kinghao <[email protected]> AuthorDate: Sat Dec 20 14:15:18 2025 +0800 mvn compile optimize --- README.md | 13 ++++ README_CN.md | 13 ++++ hybrid-build.cmd | 159 +++++++++++++++++++++++++++++++++++++++++++++ hybrid-build.ps1 | 184 +++++++++++++++++++++++++++++++++++++++++++++++++++++ hybrid-build.sh | 191 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 560 insertions(+) diff --git a/README.md b/README.md index e378297288..c52f2d9f4d 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,19 @@ Note: If you want use `-Dlinkis.build.web=true` to build linkis-web image, you mvnw.cmd -N install mvnw.cmd clean install -Dmaven.javadoc.skip=true -Dmaven.test.skip=true +### Hybrid Build (Recommended for faster compilation) + +The hybrid build method can significantly improve compilation speed (40-50% faster) while ensuring complete build artifacts. + +# Linux/macOS +./hybrid-build.sh + +# Windows CMD +hybrid-build.cmd + +# Windows PowerShell +.\hybrid-build.ps1 + ## compile web cd linkis/linkis-web npm install diff --git a/README_CN.md b/README_CN.md index 545e53a050..10ec9063ec 100644 --- a/README_CN.md +++ b/README_CN.md @@ -133,6 +133,19 @@ Apache Linkis | DeepWiki : https://deepwiki.com/apache/linkis mvnw.cmd -N install mvnw.cmd clean install -Dmaven.javadoc.skip=true -Dmaven.test.skip=true +### 混合编译 (推荐，编译速度更快) + +混合编译方式可以显著提升编译速度 (提升 40-50%)，同时确保编译产物完整。 + +# Linux/macOS +./hybrid-build.sh + +# Windows CMD +hybrid-build.cmd + +# Windows PowerShell +.\hybrid-build.ps1 + ## 管理台编译 cd linkis/linkis-web diff --git a/hybrid-build.cmd b/hybrid-build.cmd new file mode 100644 index 0000000000..f9aa78051f --- /dev/null +++ b/hybrid-build.cmd @@ -0,0 +1,159 @@ +@echo off +chcp 65001 >nul 2>&1 +setlocal EnableDelayedExpansion + +REM ============================================================ +REM Linkis 混合编译脚本 (Windows 版本) +REM +REM 解决方案: 先并行编译所有模块，再串行打包 linkis-dist +REM 这样既能获得并行编译的性能提升，又能保证产物完整性 +REM +REM 预期效果: 性能提升 40-50%%，产物与串行编译完全一致 +REM ============================================================ + +set "THREADS=1C" + +echo. +echo ╔════════════════════════════════════════════════════════╗ +echo ║ Linkis 混合编译模式 (Hybrid Build) ║ +echo ╚════════════════════════════════════════════════════════╝ +echo. +echo 编译策略: +echo [1/2] 并行编译所有模块 (跳过 linkis-dist) - 使用 -T %THREADS% +echo [2/2] 串行打包 linkis-dist - 确保产物完整 +echo. +echo 开始时间: %date% %time% +echo. + +REM 记录开始时间 +set "START_TIME=%time%" +call :GetSeconds "%START_TIME%" START_SECONDS + +REM ============================================================ +REM Step 1: 并行编译所有模块（跳过 linkis-dist） +REM ============================================================ +echo [1/2] 并行编译所有模块... +echo 执行: mvn clean install -T %THREADS% -DskipTests -pl "!:linkis-dist" +echo. + +set "STEP1_START=%time%" +call :GetSeconds "%STEP1_START%" STEP1_START_SEC + +call mvn clean install -T %THREADS% -DskipTests -pl "!:linkis-dist" +if %ERRORLEVEL% neq 0 ( + echo. + echo [错误] 步骤 1 编译失败! + exit /b 1 +) + +set "STEP1_END=%time%" +call :GetSeconds "%STEP1_END%" STEP1_END_SEC +set /a "STEP1_TIME=STEP1_END_SEC-STEP1_START_SEC" +if !STEP1_TIME! lss 0 set /a "STEP1_TIME+=86400" +set /a "STEP1_MIN=STEP1_TIME/60" +set /a "STEP1_SEC=STEP1_TIME%%60" + +echo. +echo [OK] 步骤 1 完成! 耗时: !STEP1_TIME! 秒 (!STEP1_MIN!分!STEP1_SEC!秒) +echo. + +REM ============================================================ +REM Step 2: 串行编译 linkis-dist +REM ============================================================ +echo [2/2] 串行打包 linkis-dist... +echo 执行: mvn install -pl :linkis-dist -DskipTests +echo. + +set "STEP2_START=%time%" +call :GetSeconds "%STEP2_START%" STEP2_START_SEC + +call mvn install -pl :linkis-dist -DskipTests +if %ERRORLEVEL% neq 0 ( + echo. + echo [错误] 步骤 2 编译失败! + exit /b 1 +) + +set "STEP2_END=%time%" +call :GetSeconds "%STEP2_END%" STEP2_END_SEC +set /a "STEP2_TIME=STEP2_END_SEC-STEP2_START_SEC" +if !STEP2_TIME! lss 0 set /a "STEP2_TIME+=86400" +set /a "STEP2_MIN=STEP2_TIME/60" +set /a "STEP2_SEC=STEP2_TIME%%60" + +echo. +echo [OK] 步骤 2 完成! 耗时: !STEP2_TIME! 秒 (!STEP2_MIN!分!STEP2_SEC!秒) +echo. + +REM ============================================================ +REM 计算总时间并显示结果 +REM ============================================================ +set "END_TIME=%time%" +call :GetSeconds "%END_TIME%" END_SECONDS +set /a "TOTAL_TIME=END_SECONDS-START_SECONDS" +if !TOTAL_TIME! lss 0 set /a "TOTAL_TIME+=86400" +set /a "TOTAL_MIN=TOTAL_TIME/60" +set /a "TOTAL_SEC=TOTAL_TIME%%60" + +echo ╔════════════════════════════════════════════════════════╗ +echo ║ 编译完成! ║ +echo ╚════════════════════════════════════════════════════════╝ +echo. +echo 耗时统计: +echo 步骤 1 (并行编译模块): !STEP1_TIME! 秒 (!STEP1_MIN!分!STEP1_SEC!秒) +echo 步骤 2 (串行打包): !STEP2_TIME! 秒 (!STEP2_MIN!分!STEP2_SEC!秒) +echo ──────────────────────────── +echo 总耗时: !TOTAL_TIME! 秒 (!TOTAL_MIN!分!TOTAL_SEC!秒) +echo. + +REM 检查产物 +set "DIST_DIR=linkis-dist\target\apache-linkis-1.8.0-bin" +if exist "%DIST_DIR%" ( + echo 产物信息: + echo 目录: %DIST_DIR% + + REM 统计文件数 + set "FILE_COUNT=0" + for /r "%DIST_DIR%" %%f in (*) do set /a "FILE_COUNT+=1" + echo 文件数: !FILE_COUNT! + echo. + + REM 检查关键目录 + echo 关键模块检查: + if exist "%DIST_DIR%\linkis-package\lib\linkis-computation-governance\linkis-cg-engineconnmanager" ( + echo [OK] linkis-cg-engineconnmanager + ) else ( + echo [X] linkis-cg-engineconnmanager (缺失!) + ) + if exist "%DIST_DIR%\linkis-package\lib\linkis-computation-governance\linkis-cg-entrance" ( + echo [OK] linkis-cg-entrance + ) else ( + echo [X] linkis-cg-entrance (缺失!) + ) + if exist "%DIST_DIR%\linkis-package\lib\linkis-computation-governance\linkis-cg-linkismanager" ( + echo [OK] linkis-cg-linkismanager + ) else ( + echo [X] linkis-cg-linkismanager (缺失!) + ) + echo. +) + +echo 混合编译完成! +echo 结束时间: %date% %time% +echo. + +exit /b 0 + +REM ============================================================ +REM 函数: 将时间转换为秒数 +REM ============================================================ +:GetSeconds +set "TIME_STR=%~1" +REM 处理时间格式 HH:MM:SS.CC 或 H:MM:SS.CC +for /f "tokens=1-4 delims=:,." %%a in ("%TIME_STR%") do ( + set /a "HOURS=%%a" + set /a "MINS=%%b" + set /a "SECS=%%c" +) +set /a "%~2=HOURS*3600+MINS*60+SECS" +exit /b diff --git a/hybrid-build.ps1 b/hybrid-build.ps1 new file mode 100644 index 0000000000..19de2010a3 --- /dev/null +++ b/hybrid-build.ps1 @@ -0,0 +1,184 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Linkis 混合编译脚本 (PowerShell 版本) + +.DESCRIPTION + 解决方案: 先并行编译所有模块，再串行打包 linkis-dist + 这样既能获得并行编译的性能提升，又能保证产物完整性 + 预期效果: 性能提升 40-50%，产物与串行编译完全一致 + +.PARAMETER Threads + 并行编译线程数，默认为 "1C" (使用 CPU 核心数) + +.PARAMETER SkipTests + 是否跳过测试，默认为 $true + +.EXAMPLE + .\hybrid-build.ps1 + +.EXAMPLE + .\hybrid-build.ps1 -Threads 4 +#> + +param( + [string]$Threads = "1C", + [switch]$SkipTests = $true +) + +$ErrorActionPreference = "Stop" +$OutputEncoding = [System.Text.Encoding]::UTF8 + +# 颜色定义 +function Write-ColorOutput { + param( + [string]$Message, + [string]$Color = "White" + ) + Write-Host $Message -ForegroundColor $Color +} + +function Write-Banner { + param([string]$Text) + Write-Host "" + Write-ColorOutput "╔════════════════════════════════════════════════════════╗" "Cyan" + Write-ColorOutput "║$($Text.PadLeft(29 + $Text.Length/2).PadRight(58))║" "Cyan" + Write-ColorOutput "╚════════════════════════════════════════════════════════╝" "Cyan" + Write-Host "" +} + +function Format-Duration { + param([TimeSpan]$Duration) + if ($Duration.TotalMinutes -ge 1) { + return "{0}分{1}秒" -f [int]$Duration.TotalMinutes, $Duration.Seconds + } + return "{0}秒" -f [int]$Duration.TotalSeconds +} + +# ============================================================ +# 主程序开始 +# ============================================================ + +Write-Banner "Linkis 混合编译模式 (Hybrid Build)" + +Write-ColorOutput "📋 编译策略:" "Yellow" +Write-Host " [1/2] 并行编译所有模块 (跳过 linkis-dist) - 使用 -T $Threads" +Write-Host " [2/2] 串行打包 linkis-dist - 确保产物完整" +Write-Host "" +Write-ColorOutput ("⏱️ 开始时间: " + (Get-Date -Format "yyyy-MM-dd HH:mm:ss")) "Yellow" +Write-Host "" + +$TotalStartTime = Get-Date +$SkipTestsArg = if ($SkipTests) { "-DskipTests" } else { "" } + +# ============================================================ +# Step 1: 并行编译所有模块（跳过 linkis-dist） +# ============================================================ +Write-ColorOutput "[1/2] 🚀 并行编译所有模块..." "Green" +$cmd = "mvn clean install -T $Threads $SkipTestsArg -pl `"!:linkis-dist`"" +Write-Host "执行: $cmd" +Write-Host "" + +$Step1Start = Get-Date + +try { + & mvn clean install -T $Threads $SkipTestsArg -pl "!:linkis-dist" + if ($LASTEXITCODE -ne 0) { + throw "Maven 编译失败，退出码: $LASTEXITCODE" + } +} catch { + Write-ColorOutput "❌ 步骤 1 编译失败: $_" "Red" + exit 1 +} + +$Step1End = Get-Date +$Step1Duration = $Step1End - $Step1Start + +Write-Host "" +Write-ColorOutput ("✅ 步骤 1 完成! 耗时: " + (Format-Duration $Step1Duration)) "Green" +Write-Host "" + +# ============================================================ +# Step 2: 串行编译 linkis-dist +# ============================================================ +Write-ColorOutput "[2/2] 📦 串行打包 linkis-dist..." "Green" +$cmd = "mvn install -pl :linkis-dist $SkipTestsArg" +Write-Host "执行: $cmd" +Write-Host "" + +$Step2Start = Get-Date + +try { + & mvn install -pl :linkis-dist $SkipTestsArg + if ($LASTEXITCODE -ne 0) { + throw "Maven 打包失败，退出码: $LASTEXITCODE" + } +} catch { + Write-ColorOutput "❌ 步骤 2 编译失败: $_" "Red" + exit 1 +} + +$Step2End = Get-Date +$Step2Duration = $Step2End - $Step2Start + +Write-Host "" +Write-ColorOutput ("✅ 步骤 2 完成! 耗时: " + (Format-Duration $Step2Duration)) "Green" +Write-Host "" + +# ============================================================ +# 显示结果 +# ============================================================ +$TotalEndTime = Get-Date +$TotalDuration = $TotalEndTime - $TotalStartTime + +Write-Banner "编译完成!" + +Write-ColorOutput "📊 耗时统计:" "Yellow" +Write-Host (" 步骤 1 (并行编译模块): " + (Format-Duration $Step1Duration)) +Write-Host (" 步骤 2 (串行打包): " + (Format-Duration $Step2Duration)) +Write-Host " ────────────────────────────" +Write-ColorOutput (" 总耗时: " + (Format-Duration $TotalDuration)) "Green" +Write-Host "" + +# 检查产物 +$DistDir = Join-Path $PSScriptRoot "linkis-dist\target\apache-linkis-1.8.0-bin" +if (Test-Path $DistDir) { + $Files = Get-ChildItem -Path $DistDir -Recurse -File + $FileCount = $Files.Count + $TotalSize = ($Files | Measure-Object -Property Length -Sum).Sum + $SizeFormatted = if ($TotalSize -ge 1GB) { + "{0:N2} GB" -f ($TotalSize / 1GB) + } elseif ($TotalSize -ge 1MB) { + "{0:N0} MB" -f ($TotalSize / 1MB) + } else { + "{0:N0} KB" -f ($TotalSize / 1KB) + } + + Write-ColorOutput "📦 产物信息:" "Yellow" + Write-Host " 目录: $DistDir" + Write-Host " 文件数: $FileCount" + Write-Host " 总大小: $SizeFormatted" + Write-Host "" + + # 检查关键目录 + Write-ColorOutput "🔍 关键模块检查:" "Yellow" + $Modules = @( + "linkis-cg-engineconnmanager", + "linkis-cg-entrance", + "linkis-cg-linkismanager" + ) + + foreach ($Module in $Modules) { + $ModulePath = Join-Path $DistDir "linkis-package\lib\linkis-computation-governance\$Module" + if (Test-Path $ModulePath) { + Write-ColorOutput " ✅ $Module" "Green" + } else { + Write-ColorOutput " ❌ $Module (缺失!)" "Red" + } + } + Write-Host "" +} + +Write-ColorOutput "🎉 混合编译完成!" "Green" +Write-Host (" 结束时间: " + (Get-Date -Format "yyyy-MM-dd HH:mm:ss")) +Write-Host "" diff --git a/hybrid-build.sh b/hybrid-build.sh new file mode 100755 index 0000000000..87d88e6f6e --- /dev/null +++ b/hybrid-build.sh @@ -0,0 +1,191 @@ +#!/bin/bash +# +# Linkis 混合编译脚本 (Linux/macOS) +# +# 解决方案: 先并行编译所有模块，再串行打包 linkis-dist +# 这样既能获得并行编译的性能提升，又能保证产物完整性 +# +# 预期效果: 性能提升 40-50%，产物与串行编译完全一致 +# +# 用法: ./hybrid-build.sh [选项] +# -t, --threads <N> 并行线程数，默认为 1C (CPU核心数) +# -s, --skip-tests 跳过测试 (默认) +# -r, --run-tests 运行测试 +# -h, --help 显示帮助 +# + +set -e + +# 默认参数 +THREADS="1C" +SKIP_TESTS=true + +# 颜色定义 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +# 获取脚本所在目录 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$SCRIPT_DIR" + +# 帮助信息 +show_help() { + echo "用法: $0 [选项]" + echo "" + echo "选项:" + echo " -t, --threads <N> 并行线程数，默认为 1C (CPU核心数)" + echo " -s, --skip-tests 跳过测试 (默认)" + echo " -r, --run-tests 运行测试" + echo " -h, --help 显示帮助" + echo "" + echo "示例:" + echo " $0 使用默认设置编译" + echo " $0 -t 4 使用 4 线程编译" + echo " $0 -r 运行测试" +} + +# 解析参数 +while [[ $# -gt 0 ]]; do + case $1 in + -t|--threads) + THREADS="$2" + shift 2 + ;; + -s|--skip-tests) + SKIP_TESTS=true + shift + ;; + -r|--run-tests) + SKIP_TESTS=false + shift + ;; + -h|--help) + show_help + exit 0 + ;; + *) + echo "未知选项: $1" + show_help + exit 1 + ;; + esac +done + +# 构建 Maven 参数 +SKIP_TESTS_ARG="" +if [ "$SKIP_TESTS" = true ]; then + SKIP_TESTS_ARG="-DskipTests" +fi + +# 格式化时间 +format_duration() { + local seconds=$1 + local minutes=$((seconds / 60)) + local secs=$((seconds % 60)) + if [ $minutes -gt 0 ]; then + echo "${minutes}分${secs}秒" + else + echo "${secs}秒" + fi +} + +echo "" +echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}" +echo -e "${BLUE}║ Linkis 混合编译模式 (Hybrid Build) ║${NC}" +echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}" +echo "" +echo -e "${YELLOW}📋 编译策略:${NC}" +echo " [1/2] 并行编译所有模块 (跳过 linkis-dist) - 使用 -T $THREADS" +echo " [2/2] 串行打包 linkis-dist - 确保产物完整" +echo "" +echo -e "${YELLOW}⏱️ 开始时间: $(date '+%Y-%m-%d %H:%M:%S')${NC}" +echo "" + +# 记录开始时间 +START_TIME=$(date +%s) + +# ============================================================ +# Step 1: 并行编译所有模块（跳过 linkis-dist） +# ============================================================ +echo -e "${GREEN}[1/2] 🚀 并行编译所有模块...${NC}" +echo "执行: mvn clean install -T $THREADS $SKIP_TESTS_ARG -pl '!:linkis-dist'" +echo "" + +cd "$PROJECT_DIR" +STEP1_START=$(date +%s) + +mvn clean install -T $THREADS $SKIP_TESTS_ARG -pl '!:linkis-dist' + +STEP1_END=$(date +%s) +STEP1_TIME=$((STEP1_END - STEP1_START)) + +echo "" +echo -e "${GREEN}✅ 步骤 1 完成! 耗时: ${STEP1_TIME} 秒 ($(format_duration $STEP1_TIME))${NC}" +echo "" + +# ============================================================ +# Step 2: 串行编译 linkis-dist +# ============================================================ +echo -e "${GREEN}[2/2] 📦 串行打包 linkis-dist...${NC}" +echo "执行: mvn install -pl :linkis-dist $SKIP_TESTS_ARG" +echo "" + +STEP2_START=$(date +%s) + +mvn install -pl :linkis-dist $SKIP_TESTS_ARG + +STEP2_END=$(date +%s) +STEP2_TIME=$((STEP2_END - STEP2_START)) + +echo "" +echo -e "${GREEN}✅ 步骤 2 完成! 耗时: ${STEP2_TIME} 秒 ($(format_duration $STEP2_TIME))${NC}" +echo "" + +# ============================================================ +# 显示结果 +# ============================================================ +END_TIME=$(date +%s) +TOTAL_TIME=$((END_TIME - START_TIME)) + +echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}" +echo -e "${BLUE}║ 编译完成! ║${NC}" +echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}" +echo "" +echo -e "${YELLOW}📊 耗时统计:${NC}" +echo " 步骤 1 (并行编译模块): ${STEP1_TIME} 秒 ($(format_duration $STEP1_TIME))" +echo " 步骤 2 (串行打包): ${STEP2_TIME} 秒 ($(format_duration $STEP2_TIME))" +echo " ────────────────────────────" +echo -e " ${GREEN}总耗时: ${TOTAL_TIME} 秒 ($(format_duration $TOTAL_TIME))${NC}" +echo "" + +# 检查产物 +DIST_DIR="$PROJECT_DIR/linkis-dist/target/apache-linkis-1.8.0-bin" +if [ -d "$DIST_DIR" ]; then + FILE_COUNT=$(find "$DIST_DIR" -type f 2>/dev/null | wc -l) + DIR_SIZE=$(du -sh "$DIST_DIR" 2>/dev/null | cut -f1) + + echo -e "${YELLOW}📦 产物信息:${NC}" + echo " 目录: $DIST_DIR" + echo " 文件数: $FILE_COUNT" + echo " 总大小: $DIR_SIZE" + echo "" + + # 检查关键目录 + echo -e "${YELLOW}🔍 关键模块检查:${NC}" + for module in "linkis-cg-engineconnmanager" "linkis-cg-entrance" "linkis-cg-linkismanager"; do + module_path="$DIST_DIR/linkis-package/lib/linkis-computation-governance/$module" + if [ -d "$module_path" ]; then + echo -e " ${GREEN}✅ $module${NC}" + else + echo -e " ${RED}❌ $module (缺失!)${NC}" + fi + done + echo "" +fi + +echo -e "${GREEN}🎉 混合编译完成!${NC}" +echo " 结束时间: $(date '+%Y-%m-%d %H:%M:%S')" +echo "" --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
