This is an automated email from the ASF dual-hosted git repository. nightowl888 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/lucenenet.git
commit d484b330b38887d352473309ba14b2a9ff5ca5ae Author: Michael <[email protected]> AuthorDate: Wed Jul 1 00:45:05 2020 +0700 build improvements. Pipeline caching. --- azure-pipelines.yml | 143 +++++++++++------------ build/azure-templates/publish-nuget-packages.yml | 16 +-- build/azure-templates/run-tests-on-os.yml | 33 +++--- build/build.ps1 | 16 +-- 4 files changed, 94 insertions(+), 114 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 27f0469..4fd83bb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -70,7 +70,9 @@ variables: - name: NuGetArtifactDirectory # NuGet binaries directory value: '$(Build.ArtifactStagingDirectory)/$(NuGetArtifactName)' - name: PublishedArtifactZipFileName - value: 'published.zip' + value: 'published.zip' +- name: NUGET_PACKAGES + value: $(Build.SourcesDirectory)\.nuget\packages stages: - stage: Build_Stage @@ -81,19 +83,28 @@ stages: pool: vmImage: 'windows-2019' + variables: PublishTempDirectory: '$(Build.BinariesDirectory)/publish' steps: - - powershell: | - $configuration = if ($env:BUILDCONFIGURATION) { $env:BUILDCONFIGURATION } else { "Release" } - Write-Host "##vso[task.setvariable variable=BuildConfiguration;]$configuration" - displayName: 'Setup Default Variable Values' + + - checkout: self # self represents the repo where the initial Pipelines YAML file was found + fetchDepth: 1 # the depth of commits to ask Git to fetch - task: UseDotNet@2 displayName: 'Use .NET Core sdk 3.1.301' inputs: version: 3.1.301 + + - task: Cache@2 + displayName: 'Cache NuGet packages' + inputs: + key: 'nuget | "$(Agent.OS)" | **/*.csproj' + restoreKeys: | + nuget | "$(Agent.OS)" + nuget + path: $(NUGET_PACKAGES) - powershell: | Import-Module "$(BuildDirectory)/psake.psm1" @@ -113,33 +124,8 @@ stages: Invoke-psake $(BuildDirectory)/build.ps1 -Task $tasks -properties $properties -parameters $parameters exit !($psake.build_success) displayName: 'PSake Build, Pack, and Publish' - - template: 'build/azure-templates/show-all-environment-variables.yml' - - - task: ArchiveFiles@2 - displayName: 'Zip $(PublishTempDirectory)/netcoreapp3.1' - inputs: - rootFolderOrFile: '$(PublishTempDirectory)/netcoreapp3.1' - includeRootFolder: false - archiveFile: '$(PublishDirectory)/netcoreapp3.1/$(PublishedArtifactZipFileName)' - condition: and(succeeded(), ne(variables['RunTests'], 'false')) - - - task: ArchiveFiles@2 - displayName: 'Zip $(PublishTempDirectory)/netcoreapp2.1' - inputs: - rootFolderOrFile: '$(PublishTempDirectory)/netcoreapp2.1' - includeRootFolder: false - archiveFile: '$(PublishDirectory)/netcoreapp2.1/$(PublishedArtifactZipFileName)' - condition: and(succeeded(), ne(variables['RunTests'], 'false')) - - - task: ArchiveFiles@2 - displayName: 'Zip $(PublishTempDirectory)/net48' - inputs: - rootFolderOrFile: '$(PublishTempDirectory)/net48' - includeRootFolder: false - archiveFile: '$(PublishDirectory)/net48/$(PublishedArtifactZipFileName)' - condition: and(succeeded(), ne(variables['RunTests'], 'false')) - + - powershell: | $dir = '$(Build.ArtifactStagingDirectory)/$(VersionArtifactName)' if (!(Test-Path $dir)) { New-Item -ItemType Directory -Path "$dir" -Force } @@ -147,7 +133,6 @@ stages: '$(FileVersion)' | Out-File -FilePath "$dir/$(FileVersionFileName)" -Force '$(Build.BuildNumber)' | Out-File -FilePath "$dir/$(BuildNumberFileName)" -Force displayName: 'Write Versions to Files' - # If this is a release pipeline, copy the build.bat and Version.props files as version artifacts, which will # overwrite the build.bat and Version.props files of the release. - task: CopyFiles@2 @@ -159,41 +144,47 @@ stages: Version.props TargetFolder: '$(Build.ArtifactStagingDirectory)/$(VersionArtifactName)' condition: and(succeeded(), eq(variables['IsRelease'], 'true')) - - - task: PublishBuildArtifacts@1 + + - task: PublishPipelineArtifact@1 displayName: 'Publish Artifact: $(VersionArtifactName)' inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/$(VersionArtifactName)' - ArtifactName: '$(VersionArtifactName)' - + targetPath: '$(Build.ArtifactStagingDirectory)/$(VersionArtifactName)' + artifact: '$(VersionArtifactName)' + publishLocation: 'pipeline' + # Copy the .pdb files as build artifacts, which will # later be used to push to the Azure Artifacts symbol server. - task: CopyFiles@2 displayName: 'Copy .pdb Files to: /$(DebugArtifactName)' inputs: SourceFolder: '$(System.DefaultWorkingDirectory)' - Contents: '**/bin/$(BuildConfiguration)/**/*.pdb' + Contents: | + **\**\bin\**\*.pdb TargetFolder: '$(Build.ArtifactStagingDirectory)/$(DebugArtifactName)' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: $(BinaryArtifactName)' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: $(BinaryArtifactName)_netcoreapp3.1' inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/$(BinaryArtifactName)' - ArtifactName: '$(BinaryArtifactName)' + targetPath: '$(PublishTempDirectory)\netcoreapp3.1' + artifact: '$(BinaryArtifactName)_netcoreapp3.1' + publishLocation: 'pipeline' condition: and(succeeded(), ne(variables['RunTests'], 'false')) - - - task: PublishBuildArtifacts@1 + + + - task: PublishPipelineArtifact@1 displayName: 'Publish Artifact: $(NuGetArtifactName)' inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/$(NuGetArtifactName)' - ArtifactName: '$(NuGetArtifactName)' - - - task: PublishBuildArtifacts@1 + targetPath: '$(Build.ArtifactStagingDirectory)/$(NuGetArtifactName)' + artifact: '$(NuGetArtifactName)' + publishLocation: 'pipeline' + + - task: PublishPipelineArtifact@1 displayName: 'Publish Artifact: $(DebugArtifactName)' inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/$(DebugArtifactName)' - ArtifactName: '$(DebugArtifactName)' - + targetPath: '$(Build.ArtifactStagingDirectory)/$(DebugArtifactName)' + artifact: '$(DebugArtifactName)' + publishLocation: 'pipeline' + - job: Docs condition: and(succeeded(), eq(variables['GenerateDocs'], 'true')) pool: @@ -323,7 +314,6 @@ stages: maximumParallelJobs: 9 maximumAllowedFailures: 4 # Maximum allowed failures for a successful build - - stage: Publish_Stage displayName: 'Publish Stage:' jobs: @@ -336,22 +326,27 @@ stages: vmImage: 'windows-2019' steps: + - checkout: none # self represents the repo where the initial Pipelines YAML file was found - template: 'build/azure-templates/show-all-environment-variables.yml' - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@0 displayName: 'Download Build Artifacts: $(VersionArtifactName)' inputs: artifactName: '$(VersionArtifactName)' - downloadPath: '$(Build.ArtifactStagingDirectory)' + targetPath: version + + - powershell: | + Get-ChildItem -Path $(System.DefaultWorkingDirectory) + - powershell: | + Get-ChildItem -Path version # NOTE: We are setting Build.BuildNumber here to the NuGet package version to work around the limitation that # the version cannot be passed to the Index Sources & Publish Symbols task. - powershell: | - $version = Get-Content '$(Build.ArtifactStagingDirectory)/$(VersionArtifactName)/$(PackageVersionFileName)' -Raw + $version = Get-Content '$(VersionArtifactName)/$(PackageVersionFileName)' -Raw Write-Host "##vso[task.setvariable variable=PackageVersion;]$version" Write-Host "##vso[build.updatebuildnumber]$version" displayName: 'Read PackageVersion from File to Build.BuildNumber' - - template: 'build/azure-templates/show-all-environment-variables.yml' - template: 'build/azure-templates/publish-nuget-packages.yml' @@ -360,7 +355,6 @@ stages: nugetArtifactName: '$(NuGetArtifactName)' debugArtifactName: '$(DebugArtifactName)' - - stage: Release_Stage displayName: 'Release Stage:' jobs: @@ -371,39 +365,40 @@ stages: vmImage: 'windows-2019' steps: + - checkout: none # self represents the repo where the initial Pipelines YAML file was found - template: 'build/azure-templates/show-all-environment-variables.yml' - - task: DownloadBuildArtifacts@0 + + - task: DownloadPipelineArtifact@0 displayName: 'Download Build Artifacts: $(NuGetArtifactName)' inputs: artifactName: '$(NuGetArtifactName)' - downloadPath: '$(Build.ArtifactStagingDirectory)' + targetPath: nuget + - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@0 displayName: 'Download Build Artifacts: $(VersionArtifactName)' inputs: artifactName: '$(VersionArtifactName)' - downloadPath: '$(Build.ArtifactStagingDirectory)' - + targetPath: version + - template: 'build/azure-templates/show-all-files.yml' # Uncomment for debugging # NOTE: We are setting Build.BuildNumber here to the NuGet package version to work around the limitation that # the version cannot be passed to the Index Sources & Publish Symbols task. - powershell: | - $version = Get-Content '$(Build.ArtifactStagingDirectory)/$(VersionArtifactName)/$(PackageVersionFileName)' -Raw + $version = Get-Content '$(VersionArtifactName)/$(PackageVersionFileName)' -Raw $vcsLabel = 'Lucene.Net_' + $version.Replace('.', '_').Replace('-', '_') Write-Host "##vso[task.setvariable variable=VCSLabel;]$vcsLabel" Write-Host "##vso[task.setvariable variable=PackageVersion;]$version" Write-Host "##vso[build.updatebuildnumber]$version" displayName: 'Build VCS Label and Rehydrate Version Variables' - - powershell: | $files = 'build.bat','Version.props' foreach ($file in $files) { - Copy-Item -Path "$(Build.ArtifactStagingDirectory)/$(VersionArtifactName)/$file" -Destination "$(Build.SourcesDirectory)/$file" -Force -ErrorAction Continue + Copy-Item -Path "$(VersionArtifactName)/$file" -Destination "$(Build.SourcesDirectory)/$file" -Force -ErrorAction Continue } displayName: 'Update build.bat and Version.props to build only version $(PackageVersion)' - - template: 'build/azure-templates/show-all-environment-variables.yml' - task: CopyFiles@2 @@ -428,18 +423,18 @@ stages: archiveFile: '$(Build.ArtifactStagingDirectory)/$(ReleaseArtifactName)/Apache-Lucene.Net-$(PackageVersion).src.zip' - task: CopyFiles@2 - displayName: 'Copy License/Notice Files to: $(Build.ArtifactStagingDirectory)/$(NuGetArtifactName)' + displayName: 'Copy License/Notice Files to: $(NuGetArtifactName)' inputs: SourceFolder: '$(Build.SourcesDirectory)' Contents: | LICENSE.txt NOTICE.txt - TargetFolder: '$(Build.ArtifactStagingDirectory)/$(NuGetArtifactName)' + TargetFolder: '$(NuGetArtifactName)' - task: ArchiveFiles@2 displayName: 'Archive Binary Files' inputs: - rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/$(NuGetArtifactName)' + rootFolderOrFile: '$(NuGetArtifactName)' includeRootFolder: false archiveFile: '$(Build.ArtifactStagingDirectory)/$(ReleaseArtifactName)/Apache-Lucene.Net-$(PackageVersion).bin.zip' @@ -455,11 +450,11 @@ stages: "TODO: Push release artifacts to dev (https://dist.apache.org/repos/dist/dev/lucenenet/)" + $nl + ` "TODO: Start release [VOTE] (see https://www.apache.org/foundation/voting.html)" + $nl | Out-File -FilePath "$dir/RELEASE-TODO.txt" -Force displayName: 'Write RELEASE-TODO.txt' - - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@1 displayName: 'Publish Artifact: $(ReleaseArtifactName)' inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/$(ReleaseArtifactName)' - ArtifactName: '$(ReleaseArtifactName)' + targetPath: '$(Build.ArtifactStagingDirectory)/$(ReleaseArtifactName)' + artifact: '$(ReleaseArtifactName)' + publishLocation: 'pipeline' # LUCENENET TODO: Write VCS Label (git tag) automatically \ No newline at end of file diff --git a/build/azure-templates/publish-nuget-packages.yml b/build/azure-templates/publish-nuget-packages.yml index 83ac687..f718bcc 100644 --- a/build/azure-templates/publish-nuget-packages.yml +++ b/build/azure-templates/publish-nuget-packages.yml @@ -40,30 +40,30 @@ steps: EnsureNotNullOrEmpty('${{ parameters.testSymbolFilesConvention }}', 'testSymbolFilesConvention') displayName: 'Validate Template Parameters' -- task: DownloadBuildArtifacts@0 +- task: DownloadPipelineArtifact@0 displayName: 'Download Build Artifacts: ${{ parameters.nugetArtifactName }}' inputs: - artifactName: ${{ parameters.nugetArtifactName }} - downloadPath: '$(Build.ArtifactStagingDirectory)' + artifactName: '${{ parameters.nugetArtifactName }}' + targetPath: nuget -- task: DownloadBuildArtifacts@0 +- task: DownloadPipelineArtifact@0 displayName: 'Download Build Artifacts: ${{ parameters.debugArtifactName }}' inputs: - artifactName: ${{ parameters.debugArtifactName }} - downloadPath: '$(Build.ArtifactStagingDirectory)' + artifactName: '${{ parameters.debugArtifactName }}' + targetPath: debug - task: NuGetCommand@2 displayName: 'NuGet push' inputs: command: push - packagesToPush: '$(Build.ArtifactStagingDirectory)/${{ parameters.nugetArtifactName }}/**/*.nupkg;!$(Build.ArtifactStagingDirectory)/${{ parameters.nugetArtifactName }}/**/*.symbols.nupkg' + packagesToPush: '${{ parameters.nugetArtifactName }}/*.nupkg;!${{ parameters.nugetArtifactName }}/*.symbols.nupkg' publishVstsFeed: '/${{ parameters.artifactFeedID }}' allowPackageConflicts: true - task: PublishSymbols@2 displayName: 'Publish symbols path' inputs: - SymbolsFolder: '$(Build.ArtifactStagingDirectory)/${{ parameters.debugArtifactName }}' + SymbolsFolder: '${{ parameters.debugArtifactName }}' SearchPattern: | **/bin/**/*.pdb !${{ parameters.testSymbolFilesConvention }} diff --git a/build/azure-templates/run-tests-on-os.yml b/build/azure-templates/run-tests-on-os.yml index 5b20620..56118b7 100644 --- a/build/azure-templates/run-tests-on-os.yml +++ b/build/azure-templates/run-tests-on-os.yml @@ -31,8 +31,10 @@ parameters: maximumParallelJobs: 8 maximumAllowedFailures: 0 where: '' # A test filter expression, as defined by dotnet vstest - + steps: +- checkout: none # self represents the repo where the initial Pipelines YAML file was found + - powershell: | function EnsureNotNullOrEmpty([string]$param, [string]$nameOfParam) { if ([string]::IsNullOrEmpty($param)) { @@ -52,6 +54,15 @@ steps: EnsureNotNullOrEmpty('${{ parameters.maximumAllowedFailures }}', 'maximumAllowedFailures') displayName: 'Validate Template Parameters' + +- task: DownloadPipelineArtifact@2 + displayName: 'Download Build Artifacts: ${{ parameters.binaryArtifactName }} to $(System.DefaultWorkingDirectory)\$(parameters.framework)' + inputs: + artifactName: '${{ parameters.binaryArtifactName }}_${{ parameters.framework }}' + targetPath: ${{ parameters.framework }} + +- powershell: | + Get-ChildItem -Path $(System.DefaultWorkingDirectory) - task: UseDotNet@2 displayName: 'Use .NET Core sdk 3.1.301' inputs: @@ -64,22 +75,8 @@ steps: version: 2.1.807 condition: and(succeeded(), contains('${{ parameters.framework }}', 'netcoreapp2.')) -- task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts: ${{ parameters.binaryArtifactName }}' - inputs: - artifactName: ${{ parameters.binaryArtifactName }} - downloadPath: '$(System.DefaultWorkingDirectory)' - itemPattern: '**/${{ parameters.framework }}/*.*' - #- template: 'show-all-files.yml' # Uncomment for debugging -- task: ExtractFiles@1 - displayName: 'Extract files from ${{ parameters.publishedArtifactZipFileName }}' - inputs: - archiveFilePatterns: '${{ parameters.binaryArtifactName }}/${{ parameters.framework }}/${{ parameters.publishedArtifactZipFileName }}' - destinationFolder: '$(System.DefaultWorkingDirectory)/${{ parameters.binaryArtifactName }}/${{ parameters.framework }}/' - cleanDestinationFolder: false - - powershell: | $framework = '${{ parameters.framework }}' $testBinaryRootDirectory = "$(System.DefaultWorkingDirectory)" @@ -210,11 +207,11 @@ steps: #- template: 'show-all-files.yml' # Uncomment for debugging -- task: PublishBuildArtifacts@1 +- task: PublishPipelineArtifact@1 displayName: 'Publish Artifact: ${{ parameters.testResultsArtifactName }}' inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/${{ parameters.testResultsArtifactName }}' - ArtifactName: '${{ parameters.testResultsArtifactName }}' + targetPath: '$(Build.ArtifactStagingDirectory)/${{ parameters.testResultsArtifactName }}' + ArtifactName: '${{ parameters.testResultsArtifactName }}_${{ parameters.osName }}_${{ parameters.framework }}' condition: succeededOrFailed() # Due to the fact that it is not possible to loop a task and diff --git a/build/build.ps1 b/build/build.ps1 index ead5044..27acf2f 100644 --- a/build/build.ps1 +++ b/build/build.ps1 @@ -27,7 +27,7 @@ properties { [string]$publish_directory = "$release_directory/Publish" [string]$solutionFile = "$base_directory/Lucene.Net.sln" [string]$sdkPath = "$env:programfiles/dotnet/sdk" - [string]$sdkVersion = "3.1.100" + [string]$sdkVersion = "3.1.301" [string]$globalJsonFile = "$base_directory/global.json" [string]$versionPropsFile = "$base_directory/Version.props" [string]$build_bat = "$base_directory/build.bat" @@ -235,7 +235,7 @@ task Publish -depends Compile -description "This task uses dotnet publish to pac param([string]$testProject, [string]$outputPath, [string]$logPath, [string]$framework, [string]$configuration, [string]$projectName) Write-Host "Publishing '$testProject' on '$framework' to '$outputPath'..." # Note: Cannot use Psake Exec in background - dotnet publish "$testProject" --output "$outputPath" --framework "$framework" --configuration "$configuration" --no-build --verbosity Detailed /p:TestFrameworks=true /p:Platform="$platform" > "$logPath/$projectName-dotnet-publish.log" 2> "$logPath/$projectName-dotnet-publish-error.log" + dotnet publish "$testProject" --output "$outputPath" --framework "$framework" --configuration "$configuration" --no-build --no-restore --verbosity Detailed /p:TestFrameworks=true /p:Platform="$platform" > "$logPath/$projectName-dotnet-publish.log" 2> "$logPath/$projectName-dotnet-publish-error.log" } # Execute the jobs in parallel @@ -550,7 +550,6 @@ param( ) $fileText = "<!-- - Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information @@ -558,16 +557,13 @@ $fileText = "<!-- to you under the Apache License, Version 2.0 (the ""License""); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an ""AS IS"" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - --> <Project> <PropertyGroup Label=""Version Override Properties""> @@ -626,10 +622,8 @@ GOTO endcommentblock :: ----------------------------------------------------------------------------------- :endcommentblock setlocal enabledelayedexpansion enableextensions - set runtests=false set maximumParallelJobs=8 - FOR %%a IN (%*) DO ( FOR /f ""useback tokens=*"" %%a in ('%%a') do ( set value=%%~a @@ -638,31 +632,25 @@ FOR %%a IN (%*) DO ( IF /I !test!==-t ( set runtests=true ) - set test=!value:~0,6! IF /I !test!==--test ( set runtests=true ) - set test=!value:~0,4! IF /I !test!==-mp: ( set maximumParallelJobs=!value:~4! ) - set test=!value:~0,22! IF /I !test!==--maximumparalleljobs: ( set maximumParallelJobs=!value:~22! ) ) ) - set tasks=""Default"" if ""!runtests!""==""true"" ( set tasks=""Default,Test"" ) - powershell -ExecutionPolicy Bypass -Command ""& { Import-Module .\build\psake.psm1; Invoke-Psake .\build\build.ps1 -Task %tasks% -properties @{prepareForBuild='false';backup_files='false';maximumParalellJobs=%maximumParallelJobs%} }"" - endlocal " $dir = [System.IO.Path]::GetDirectoryName($file)
