From: Sean Brogan <sean.bro...@microsoft.com>
https://bugzilla.tianocore.org/show_bug.cgi?id=2315
Add main python script for EDK II Continuous Integration (CI)
builds along with a Readme.md that provides a summary of the
packages, platforms, and checks performs during a CI build.
Cc: Sean Brogan <sean.bro...@microsoft.com>
Cc: Bret Barkelew <bret.barke...@microsoft.com>
Cc: Liming Gao <liming....@intel.com>
Signed-off-by: Michael D Kinney <michael.d.kin...@intel.com>
---
.pytool/CISettings.py | 173 ++++++++++++++++++++++++++++++++++++++++++
.pytool/Readme.md | 128 +++++++++++++++++++++++++++++++
2 files changed, 301 insertions(+)
create mode 100644 .pytool/CISettings.py
create mode 100644 .pytool/Readme.md
diff --git a/.pytool/CISettings.py b/.pytool/CISettings.py
new file mode 100644
index 0000000000..a78e8b974c
--- /dev/null
+++ b/.pytool/CISettings.py
@@ -0,0 +1,173 @@
+# @file
+#
+# Copyright (c) 2018, Microsoft Corporation
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+import os
+import logging
+from edk2toolext.environment import shell_environment
+from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager
+from edk2toolext.invocables.edk2_setup import SetupSettingsManager,
RequiredSubmodule
+from edk2toolext.invocables.edk2_update import UpdateSettingsManager
+from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager
+from edk2toollib.utility_functions import GetHostInfo
+
+
+class Settings(CiBuildSettingsManager, UpdateSettingsManager,
SetupSettingsManager, PrEvalSettingsManager):
+
+ def __init__(self):
+ self.ActualPackages = []
+ self.ActualTargets = []
+ self.ActualArchitectures = []
+ self.ActualToolChainTag = ""
+
+ #
#######################################################################################
#
+ # Extra CmdLine configuration
#
+ #
#######################################################################################
#
+
+ def AddCommandLineOptions(self, parserObj):
+ pass
+
+ def RetrieveCommandLineOptions(self, args):
+ pass
+
+ #
#######################################################################################
#
+ # Default Support for this Ci Build
#
+ #
#######################################################################################
#
+
+ def GetPackagesSupported(self):
+ ''' return iterable of edk2 packages supported by this build.
+ These should be edk2 workspace relative paths '''
+
+ return ("MdePkg",
+ "MdeModulePkg",
+ "NetworkPkg",
+ "PcAtChipsetPkg",
+ "SecurityPkg",
+ "UefiCpuPkg",
+ "FmpDevicePkg",
+ "ShellPkg",
+ "FatPkg",
+ "CryptoPkg"
+ )
+
+ def GetArchitecturesSupported(self):
+ ''' return iterable of edk2 architectures supported by this build '''
+ return ("IA32",
+ "X64",
+ "ARM",
+ "AARCH64")
+
+ def GetTargetsSupported(self):
+ ''' return iterable of edk2 target tags supported by this build '''
+ return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")
+
+ #
#######################################################################################
#
+ # Verify and Save requested Ci Build Config
#
+ #
#######################################################################################
#
+
+ def SetPackages(self, list_of_requested_packages):
+ ''' Confirm the requested package list is valid and configure
SettingsManager
+ to build the requested packages.
+
+ Raise UnsupportedException if a requested_package is not supported
+ '''
+ unsupported = set(list_of_requested_packages) - \
+ set(self.GetPackagesSupported())
+ if(len(unsupported) > 0):
+ logging.critical(
+ "Unsupported Package Requested: " + " ".join(unsupported))
+ raise Exception("Unsupported Package Requested: " +
+ " ".join(unsupported))
+ self.ActualPackages = list_of_requested_packages
+
+ def SetArchitectures(self, list_of_requested_architectures):
+ ''' Confirm the requests architecture list is valid and configure
SettingsManager
+ to run only the requested architectures.
+
+ Raise Exception if a list_of_requested_architectures is not supported
+ '''
+ unsupported = set(list_of_requested_architectures) - \
+ set(self.GetArchitecturesSupported())
+ if(len(unsupported) > 0):
+ logging.critical(
+ "Unsupported Architecture Requested: " + " ".join(unsupported))
+ raise Exception(
+ "Unsupported Architecture Requested: " + " ".join(unsupported))
+ self.ActualArchitectures = list_of_requested_architectures
+
+ def SetTargets(self, list_of_requested_target):
+ ''' Confirm the request target list is valid and configure
SettingsManager
+ to run only the requested targets.
+
+ Raise UnsupportedException if a requested_target is not supported
+ '''
+ unsupported = set(list_of_requested_target) - \
+ set(self.GetTargetsSupported())
+ if(len(unsupported) > 0):
+ logging.critical(
+ "Unsupported Targets Requested: " + " ".join(unsupported))
+ raise Exception("Unsupported Targets Requested: " +
+ " ".join(unsupported))
+ self.ActualTargets = list_of_requested_target
+
+ #
#######################################################################################
#
+ # Actual Configuration for Ci Build
#
+ #
#######################################################################################
#
+
+ def GetActiveScopes(self):
+ ''' return tuple containing scopes that should be active for this
process '''
+ scopes = ("cibuild","edk2-build")
+
+ self.ActualToolChainTag =
shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")
+
+ if GetHostInfo().os.upper() == "LINUX" and
self.ActualToolChainTag.upper().startswith("GCC"):
+ if "AARCH64" in self.ActualArchitectures:
+ scopes += ("gcc_aarch64_linux",)
+ if "ARM" in self.ActualArchitectures:
+ scopes += ("gcc_arm_linux",)
+
+ return scopes
+
+ def GetRequiredSubmodules(self):
+ ''' return iterable containing RequiredSubmodule objects.
+ If no RequiredSubmodules return an empty iterable
+ '''
+ rs=[]
+ rs.append(RequiredSubmodule(
+ "ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))
+ rs.append(RequiredSubmodule(
+ "CryptoPkg/Library/OpensslLib/openssl", False))
+ return rs
+
+ def GetName(self):
+ return "Edk2"
+
+ def GetDependencies(self):
+ return []
+
+ def GetPackagesPath(self):
+ return ()
+
+ def GetWorkspaceRoot(self):
+ ''' get WorkspacePath '''
+ return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+ def FilterPackagesToTest(self, changedFilesList: list,
potentialPackagesList: list) -> list:
+ ''' Filter potential packages to test based on changed files. '''
+ build_these_packages=[]
+ possible_packages=potentialPackagesList.copy()
+ for f in changedFilesList:
+ nodes=f.split("/") # split each part of path for comparison later
+
+ # python file change in .pytool folder causes building all
+ if f.endswith(".py") and ".pytool" in nodes:
+ build_these_packages = possible_packages
+ break
+
+ # BaseTools files that might change the build
+ if "BaseTools" in nodes:
+ if os.path.splitext(f) not in [".txt", ".md"]:
+ build_these_packages = possible_packages
+ break
+ return build_these_packages
diff --git a/.pytool/Readme.md b/.pytool/Readme.md
new file mode 100644
index 0000000000..5c1b107213
--- /dev/null
+++ b/.pytool/Readme.md
@@ -0,0 +1,128 @@
+# Edk2 Continuous Integration
+
+## Basic Status
+
+| Package | Windows VS2019 | Ubuntu GCC5 | Known Issues |
+| :---- | :----- | :---- | :--- |
+|ArmPkg | | |
+|ArmPlatformPkg |||
+|ArmVirtPkg |||
+|CryptoPkg | :heavy_check_mark: | :heavy_check_mark: | New changes for NULL
basecryptlib and tlslib, add IntrinsicLib.inf to dsc|
+|DynamicTablesPkg |||
+|EmbeddedPkg |||
+|EmulatorPkg |||
+|FatPkg|:heavy_check_mark:|:heavy_check_mark:||
+|FmpDevicePkg|:heavy_check_mark:|:heavy_check_mark:||
+|IntelFsp2Pkg|||
+|IntelFsp2WrapperPkg|||
+| MdeModulePkg | :heavy_check_mark: |:heavy_check_mark: |DxeIpl dependency on
ArmPkg, Missing Visual Studio AARCH64/ARM support for EBC |
+|MdePkg | :heavy_check_mark: |:heavy_check_mark: | Update DSC to add
UefiFileHandleLib, Update BaseIoLibIntrinsic to support MSFT ARM/AARCH64|
+|NetworkPkg | :heavy_check_mark: |:heavy_check_mark:| Libraries missing from
components section for package dsc|
+|OvmfPkg |||
+|PcAtChipsetPkg | :heavy_check_mark: |:heavy_check_mark:|
+|SecurityPkg|:heavy_check_mark:|:heavy_check_mark:| Incorrect libraryclass in
dec|
+|ShellPkg | :heavy_check_mark: |:heavy_check_mark:||
+|SignedCapsulePkg|||
+|SourceLevelDebugPkg|||
+|StandaloneMmPkg|||
+|UefiCpuPkg | :heavy_check_mark: |:heavy_check_mark: ||
+|UefiPayloadPkg|||
+
+For more detailed status look at the test results of the latest CI run on the
repo readme.
+
+## Background
+
+While a number of CI solutions exist, this proposal will focus on the usage of
Azure Dev Ops and Build Pipelines. For demonstration, a sample [TianoCore
repo](https://github.com/tianocore/edk2-staging.git) (branch master) and [Dev
Ops
Pipeline](https://dev.azure.com/tianocore/edk2-ci-play/_build?definitionId=14)
have been set up.
+
+Furthermore, this proposal will leverage the TianoCore python tools PIP
modules: [library](https://pypi.org/project/edk2-pytool-library/) and
[extensions](https://pypi.org/project/edk2-pytool-extensions/) (with repos
located [here](https://github.com/tianocore/edk2-pytool-library) and
[here](https://github.com/tianocore/edk2-pytool-extensions)).
+
+The primary execution flows can be found in the
`ci/AzurePipelines/Windows-VS2019.yml` and `ci/AzurePipelines/Ubuntu-GCC5.yml`
files. These YAML files are consumed by the Azure Dev Ops Build Pipeline and
dictate what server resources should be used, how they should be configured,
and what processes should be run on them. An overview of this schema can be
found
[here](https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema).
+
+Inspection of these files reveals the EDKII Tools commands that make up the
primary processes for the CI build: 'stuart_setup', 'stuart_update', and
'stuart_ci_build'. These commands come from the EDKII Tools PIP modules and are
configured as described below. More documentation on the stuart tools can be
found
[here](https://github.com/tianocore/edk2-pytool-extensions/blob/master/docs/using.md)
and
[here](https://github.com/tianocore/edk2-pytool-extensions/blob/master/docs/features/feature_invocables.md).
+
+## Configuration
+
+Configuration of the CI process consists of (in order of precedence):
+
+* command-line arguments passed in via the Pipeline YAML
+* a per-package configuration file (e.g. `<package-name>.ci.yaml`) that is
detected by the CI system in EDKII Tools.
+* a global configuration Python module (e.g. `CISetting.py`) passed in via the
command-line
+
+The global configuration file is described in [this
readme](https://github.com/tianocore/edk2-pytool-extensions/blob/master/docs/usability/using_settings_manager.md)
from the EDKII Tools documentation. This configuration is written as a Python
module so that decisions can be made dynamically based on command line
parameters and codebase state.
+
+The per-package configuration file can override most settings in the global
configuration file, but is not dynamic. This file can be used to skip or
customize tests that may be incompatible with a specific package. By default,
the global configuration will try to run all tests on all packages.
+
+## Current PyTool Test Capabilities
+
+All CI tests are instances of EDKII Tools plugins. Documentation on the plugin
system can be found
[here](https://github.com/tianocore/edk2-pytool-extensions/blob/master/docs/usability/using_plugin_manager.md)
and
[here](https://github.com/tianocore/edk2-pytool-extensions/blob/master/docs/features/feature_plugin_manager.md).
Upon invocation, each plugin will be passed the path to the current package
under test and a dictionary containing its targeted configuration, as assembled
from the command line, per-package configuration, and global configuration.
+
+Note: CI plugins are considered unique from build plugins and helper plugins,
even though some CI plugins may execute steps of a build.
+
+In the example, these plugins live alongside the code under test (in the
`ci/Plugin` directory), but may be moved to the 'edk2-test' repo if that
location makes more sense for the community.
+
+### Module Inclusion Test - DscCompleteCheck
+
+This test scans all available modules (via INF files) and compares them to the
package-level DSC file for the package each module is contained within. The
test considers it an error if any module does not appear in the `Components`
section of at least one package-level DSC (indicating that it would not be
built if the package were built).
+
+### Code Compilation Test - CompilerPlugin
+
+Once the Module Inclusion Test has verified that all modules would be built if
all package-level DSCs were built, the Code Compilation Test simply runs
through and builds every package-level DSC on every toolchain and for every
architecture that is supported. Any module that fails to build is considered an
error.
+
+### GUID Uniqueness Test - GuidCheck
+
+This test works on the collection of all packages rather than an individual
package. It looks at all FILE_GUIDs and GUIDs declared in DEC files and ensures
that they are unique for the codebase. This prevents, for example, accidental
duplication of GUIDs when using an existing INF as a template for a new module.
+
+### Cross-Package Dependency Test - DependencyCheck
+
+This test compares the list of all packages used in INFs files for a given
package against a list of "allowed dependencies" in plugin configuration for
that package. Any module that depends on a disallowed package will cause a test
failure.
+
+### Library Declaration Test - LibraryClassCheck
+
+This test scans at all library header files found in the `Library` folders in
all of the package's declared include directories and ensures that all files
have a matching LibraryClass declaration in the DEC file for the package. Any
missing declarations will cause a failure.
+
+### Invalid Character Test - CharEncodingCheck
+
+This test scans all files in a package to make sure that there are no invalid
Unicode characters that may cause build errors in some character
sets/localizations.
+
+### Spell Checking - cspell
+
+This test runs a spell checker on all files within the package. This is done
using the NodeJs cspell tool. For details check `ci/Plugin/SpellCheck`. For
this plugin to run during ci you must install nodejs and cspell and have both
available to the command line when running your CI.
+
+Install
+
+* Install nodejs from https://nodejs.org/en/
+* Install cspell
+ 1. Open cmd prompt with access to node and npm
+ 2. Run `npm install -g cspell`
+
+ More cspell info: https://github.com/streetsidesoftware/cspell
+
+## Current Azure Pipeline Tests
+
+When adding a test it can be added as either a *PyTool* test or just added to
the CI build process. This should be a deliberate choice. Any change added as
a pipeline test is not as easily run on a private/local workspace. But there
are times where this is still the preferred method.
+
+## PyTool Scopes
+
+Scopes are how the PyTool ext_dep, path_env, and plugins are activated.
Meaning that if an invocable process has a scope active then those ext_dep and
path_env will be active. To allow easy integration of PyTools capabilities
there are a few standard scopes.
+
+| Scope | Invocable | Description |
+| :---- | :----- | :---- |
+| global | edk2_invocable++ - should be base_abstract_invocable |
Running an invocables |
+| global-win | edk2_invocable++ | Running on Microsoft Windows |
+| global-nix | edk2_invocable++ | Running on Linux based OS |
+| edk2-build | | This indicates that an invocable is building EDK2 based UEFI
code |
+| cibuild | set in .pytool/CISettings.py | Suggested target for edk2
continuous integration builds. Tools used for CiBuilds can use this scope.
Example: asl compiler |
+
+## Future investments
+
+* PatchCheck tests as plugins
+* MacOS/xcode support
+* Clang/LLVM support
+* Visual Studio AARCH64 and ARM support
+* BaseTools C tools CI/PR and binary release process
+* BaseTools Python tools CI/PR process
+* Host based unit testing
+* Extensible private/closed source platform reporting
+* Platform builds, validation
+* UEFI SCTs
+* Other automation
--
2.21.0.windows.1
-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.
View/Reply Online (#49603): https://edk2.groups.io/g/devel/message/49603
Mute This Topic: https://groups.io/mt/39614217/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
