Mike and Sean: I have one minor comment. Please use the same file Readme.md. I see there are two files are named as readme.md. With this change, Reviewed-by: Liming Gao <liming....@intel.com>
Thanks Liming >-----Original Message----- >From: Kinney, Michael D >Sent: Thursday, November 07, 2019 9:14 AM >To: devel@edk2.groups.io >Cc: Sean Brogan <sean.bro...@microsoft.com>; Bret Barkelew ><bret.barke...@microsoft.com>; Gao, Liming <liming....@intel.com> >Subject: [Patch v4 07/22] .pytool/Plugin: Add CI plugins > >From: Sean Brogan <sean.bro...@microsoft.com> > >Add .pytool directory to the edk2 repository with the >following plugins. These plugins are in a top level >directory because that can be used with all packages >and platforms. > >* CharEncodingCheck >* CompilerPlugin >* DependencyCheck >* DscCompleteCheck >* GuidCheck >* LibraryClassCheck >* SpellCheck > >Cc: Sean Brogan <sean.bro...@microsoft.com> >Cc: Bret Barkelew <bret.barke...@microsoft.com> >Cc: Liming Gao <liming....@intel.com> >Signed-off-by: Michael D Kinney <michael.d.kin...@intel.com> >--- > .../CharEncodingCheck/CharEncodingCheck.py | 118 ++++++++ > .../CharEncodingCheck_plug_in.yaml | 11 + > .pytool/Plugin/CharEncodingCheck/Readme.md | 13 + > .../Plugin/CompilerPlugin/CompilerPlugin.py | 102 +++++++ > .../CompilerPlugin/Compiler_plug_in.yaml | 11 + > .../Plugin/DependencyCheck/DependencyCheck.py | 120 +++++++++ > .../DependencyCheck_plug_in.yaml | 13 + > .../DscCompleteCheck/DscCompleteCheck.py | 118 ++++++++ > .../DscCompleteCheck_plug_in.yaml | 12 + > .pytool/Plugin/DscCompleteCheck/readme.md | 22 ++ > .pytool/Plugin/GuidCheck/GuidCheck.py | 251 ++++++++++++++++++ > .../Plugin/GuidCheck/GuidCheck_plug_in.yaml | 11 + > .pytool/Plugin/GuidCheck/Readme.md | 60 +++++ > .../LibraryClassCheck/LibraryClassCheck.py | 153 +++++++++++ > .../LibraryClassCheck_plug_in.yaml | 11 + > .pytool/Plugin/LibraryClassCheck/readme.md | 22 ++ > .pytool/Plugin/SpellCheck/Readme.md | 100 +++++++ > .pytool/Plugin/SpellCheck/SpellCheck.py | 216 +++++++++++++++ > .../Plugin/SpellCheck/SpellCheck_plug_in.yaml | 11 + > .pytool/Plugin/SpellCheck/cspell.base.yaml | 165 ++++++++++++ > 20 files changed, 1540 insertions(+) > create mode >100644 .pytool/Plugin/CharEncodingCheck/CharEncodingCheck.py > create mode >100644 .pytool/Plugin/CharEncodingCheck/CharEncodingCheck_plug_in.yaml > create mode 100644 .pytool/Plugin/CharEncodingCheck/Readme.md > create mode 100644 .pytool/Plugin/CompilerPlugin/CompilerPlugin.py > create mode 100644 .pytool/Plugin/CompilerPlugin/Compiler_plug_in.yaml > create mode 100644 .pytool/Plugin/DependencyCheck/DependencyCheck.py > create mode >100644 .pytool/Plugin/DependencyCheck/DependencyCheck_plug_in.yaml > create mode >100644 .pytool/Plugin/DscCompleteCheck/DscCompleteCheck.py > create mode >100644 .pytool/Plugin/DscCompleteCheck/DscCompleteCheck_plug_in.yaml > create mode 100644 .pytool/Plugin/DscCompleteCheck/readme.md > create mode 100644 .pytool/Plugin/GuidCheck/GuidCheck.py > create mode 100644 .pytool/Plugin/GuidCheck/GuidCheck_plug_in.yaml > create mode 100644 .pytool/Plugin/GuidCheck/Readme.md > create mode 100644 .pytool/Plugin/LibraryClassCheck/LibraryClassCheck.py > create mode >100644 .pytool/Plugin/LibraryClassCheck/LibraryClassCheck_plug_in.yaml > create mode 100644 .pytool/Plugin/LibraryClassCheck/readme.md > create mode 100644 .pytool/Plugin/SpellCheck/Readme.md > create mode 100644 .pytool/Plugin/SpellCheck/SpellCheck.py > create mode 100644 .pytool/Plugin/SpellCheck/SpellCheck_plug_in.yaml > create mode 100644 .pytool/Plugin/SpellCheck/cspell.base.yaml > >diff --git a/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck.py >b/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck.py >new file mode 100644 >index 0000000000..54a2424875 >--- /dev/null >+++ b/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck.py >@@ -0,0 +1,118 @@ >+# @file CharEncodingCheck.py >+# >+# Copyright (c) Microsoft Corporation. All rights reserved. >+# SPDX-License-Identifier: BSD-2-Clause-Patent >+## >+ >+ >+import os >+import logging >+from edk2toolext.environment.plugintypes.ci_build_plugin import >ICiBuildPlugin >+from edk2toolext.environment.var_dict import VarDict >+ >+## >+# map >+## >+EcodingMap = { >+ ".md": 'utf-8', >+ ".dsc": 'utf-8', >+ ".dec": 'utf-8', >+ ".c": 'utf-8', >+ ".h": 'utf-8', >+ ".asm": 'utf-8', >+ ".masm": 'utf-8', >+ ".nasm": 'utf-8', >+ ".s": 'utf-8', >+ ".inf": 'utf-8', >+ ".asl": 'utf-8', >+ ".uni": 'utf-8', >+ ".py": 'utf-8' >+} >+ >+ >+class CharEncodingCheck(ICiBuildPlugin): >+ """ >+ A CiBuildPlugin that scans each file in the code tree and confirms the >encoding is correct. >+ >+ Configuration options: >+ "CharEncodingCheck": { >+ "IgnoreFiles": [] >+ } >+ """ >+ >+ def GetTestName(self, packagename: str, environment: VarDict) -> tuple: >+ """ Provide the testcase name and classname for use in reporting >+ testclassname: a descriptive string for the testcase can include >whitespace >+ classname: should be patterned <packagename>.<plugin>.<optionally >any unique condition> >+ >+ Args: >+ packagename: string containing name of package to build >+ environment: The VarDict for the test to run in >+ Returns: >+ a tuple containing the testcase name and the classname >+ (testcasename, classname) >+ """ >+ return ("Check for valid file encoding for " + packagename, >packagename >+ ".CharEncodingCheck") >+ >+ ## >+ # External function of plugin. This function is used to perform the task >of >the ci_build_plugin Plugin >+ # >+ # - package is the edk2 path to package. This means >workspace/packagepath relative. >+ # - edk2path object configured with workspace and packages path >+ # - PkgConfig Object (dict) for the pkg >+ # - EnvConfig Object >+ # - Plugin Manager Instance >+ # - Plugin Helper Obj Instance >+ # - Junit Logger >+ # - output_stream the StringIO output stream from this plugin via >logging >+ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, >environment, PLM, PLMHelper, tc, output_stream=None): >+ overall_status = 0 >+ files_tested = 0 >+ >+ abs_pkg_path = >Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagen >ame) >+ >+ if abs_pkg_path is None: >+ tc.SetSkipped() >+ tc.LogStdError("No Package folder {0}".format(abs_pkg_path)) >+ return 0 >+ >+ for (ext, enc) in EcodingMap.items(): >+ files = self.WalkDirectoryForExtension([ext], abs_pkg_path) >+ files = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(x) for x >in files] # make edk2relative path so can process ignores >+ >+ if "IgnoreFiles" in pkgconfig: >+ for a in pkgconfig["IgnoreFiles"]: >+ a = a.replace(os.sep, "/") >+ try: >+ tc.LogStdOut("Ignoring File {0}".format(a)) >+ files.remove(a) >+ except: >+ tc.LogStdError("CharEncodingCheck.IgnoreInf -> {0} >not found in >filesystem. Invalid ignore file".format(a)) >+ logging.info("CharEncodingCheck.IgnoreInf -> {0} not >found in >filesystem. Invalid ignore file".format(a)) >+ >+ files = >[Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(x) for x in >files] >+ for a in files: >+ files_tested += 1 >+ if(self.TestEncodingOk(a, enc)): >+ logging.debug("File {0} Passed Encoding Check >{1}".format(a, enc)) >+ else: >+ tc.LogStdError("Encoding Failure in {0}. Not >{1}".format(a, enc)) >+ overall_status += 1 >+ >+ tc.LogStdOut("Tested Encoding on {0} files".format(files_tested)) >+ if overall_status is not 0: >+ tc.SetFailed("CharEncoding {0} Failed. Errors >{1}".format(packagename, >overall_status), "CHAR_ENCODING_CHECK_FAILED") >+ else: >+ tc.SetSuccess() >+ return overall_status >+ >+ def TestEncodingOk(self, apath, encodingValue): >+ try: >+ with open(apath, "rb") as fobj: >+ fobj.read().decode(encodingValue) >+ except Exception as exp: >+ logging.error("Encoding failure: file: {0} type: >{1}".format(apath, >encodingValue)) >+ logging.debug("EXCEPTION: while processing {1} - {0}".format(exp, >apath)) >+ return False >+ >+ return True >diff --git >a/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck_plug_in.yaml >b/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck_plug_in.yaml >new file mode 100644 >index 0000000000..915d3f4fdb >--- /dev/null >+++ b/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck_plug_in.yaml >@@ -0,0 +1,11 @@ >+## >+# CiBuildPlugin used to check char encoding >+# >+# Copyright (c) 2019, Microsoft Corporation >+# SPDX-License-Identifier: BSD-2-Clause-Patent >+## >+{ >+ "scope": "cibuild", >+ "name": "Char Encoding Check Test", >+ "module": "CharEncodingCheck" >+} >diff --git a/.pytool/Plugin/CharEncodingCheck/Readme.md >b/.pytool/Plugin/CharEncodingCheck/Readme.md >new file mode 100644 >index 0000000000..8350542966 >--- /dev/null >+++ b/.pytool/Plugin/CharEncodingCheck/Readme.md >@@ -0,0 +1,13 @@ >+# Character Encoding Check Plugin >+ >+This CiBuildPlugin scans all the files in a package to make sure each file is >correctly encoded and all characters can be read. Improper encoding causes >tools to fail in some situations especially in different locals. >+ >+## Configuration >+ >+The plugin can be configured to ignore certain files. >+ >+``` yaml >+"CharEncodingCheck": { >+ "IgnoreFiles": [] # optional - list of files to ignore >+} >+``` >diff --git a/.pytool/Plugin/CompilerPlugin/CompilerPlugin.py >b/.pytool/Plugin/CompilerPlugin/CompilerPlugin.py >new file mode 100644 >index 0000000000..0a357309a4 >--- /dev/null >+++ b/.pytool/Plugin/CompilerPlugin/CompilerPlugin.py >@@ -0,0 +1,102 @@ >+# @file HostUnitTestCompiler_plugin.py >+## >+# Copyright (c) Microsoft Corporation. All rights reserved. >+# SPDX-License-Identifier: BSD-2-Clause-Patent >+## >+ >+import logging >+import os >+import re >+from edk2toollib.uefi.edk2.parsers.dsc_parser import DscParser >+from edk2toolext.environment.plugintypes.ci_build_plugin import >ICiBuildPlugin >+from edk2toolext.environment.uefi_build import UefiBuilder >+from edk2toolext import edk2_logging >+from edk2toolext.environment.var_dict import VarDict >+ >+ >+class CompilerPlugin(ICiBuildPlugin): >+ """ >+ A CiBuildPlugin that compiles the package dsc >+ from the package being tested. >+ >+ Configuration options: >+ "CompilerPlugin": { >+ "DscPath": "<path to dsc from root of pkg>" >+ } >+ """ >+ >+ def GetTestName(self, packagename: str, environment: VarDict) -> tuple: >+ """ Provide the testcase name and classname for use in reporting >+ >+ Args: >+ packagename: string containing name of package to build >+ environment: The VarDict for the test to run in >+ Returns: >+ a tuple containing the testcase name and the classname >+ (testcasename, classname) >+ """ >+ target = environment.GetValue("TARGET") >+ return ("Compile " + packagename + " " + target, packagename + >".Compiler." + target) >+ >+ def RunsOnTargetList(self): >+ return ["DEBUG", "RELEASE"] >+ >+ ## >+ # External function of plugin. This function is used to perform the task >of >the MuBuild Plugin >+ # >+ # - package is the edk2 path to package. This means >workspace/packagepath relative. >+ # - edk2path object configured with workspace and packages path >+ # - PkgConfig Object (dict) for the pkg >+ # - EnvConfig Object >+ # - Plugin Manager Instance >+ # - Plugin Helper Obj Instance >+ # - Junit Logger >+ # - output_stream the StringIO output stream from this plugin via >logging >+ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, >environment, PLM, PLMHelper, tc, output_stream=None): >+ self._env = environment >+ >+ # Parse the config for required DscPath element >+ if "DscPath" not in pkgconfig: >+ tc.SetSkipped() >+ tc.LogStdError("DscPath not found in config file. Nothing to >compile.") >+ return -1 >+ >+ AP = >Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagen >ame) >+ >+ APDSC = os.path.join(AP, pkgconfig["DscPath"].strip()) >+ AP_Path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(APDSC) >+ if AP is None or AP_Path is None or not os.path.isfile(APDSC): >+ tc.SetSkipped() >+ tc.LogStdError("Package Dsc not found.") >+ return -1 >+ >+ logging.info("Building {0}".format(AP_Path)) >+ self._env.SetValue("ACTIVE_PLATFORM", AP_Path, "Set in Compiler >Plugin") >+ >+ # Parse DSC to check for SUPPORTED_ARCHITECTURES >+ dp = DscParser() >+ dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath) >+ dp.SetPackagePaths(Edk2pathObj.PackagePathList) >+ dp.ParseFile(AP_Path) >+ if "SUPPORTED_ARCHITECTURES" in dp.LocalVars: >+ SUPPORTED_ARCHITECTURES = >dp.LocalVars["SUPPORTED_ARCHITECTURES"].split('|') >+ TARGET_ARCHITECTURES = >environment.GetValue("TARGET_ARCH").split(' ') >+ >+ # Skip if there is no intersection between >SUPPORTED_ARCHITECTURES and TARGET_ARCHITECTURES >+ if len(set(SUPPORTED_ARCHITECTURES) & >set(TARGET_ARCHITECTURES)) == 0: >+ tc.SetSkipped() >+ tc.LogStdError("No supported architecutres to build") >+ return -1 >+ >+ uefiBuilder = UefiBuilder() >+ # do all the steps >+ # WorkSpace, PackagesPath, PInHelper, PInManager >+ ret = uefiBuilder.Go(Edk2pathObj.WorkspacePath, >os.pathsep.join(Edk2pathObj.PackagePathList), PLMHelper, PLM) >+ if ret != 0: # failure: >+ tc.SetFailed("Compile failed for {0}".format(packagename), >"Compile_FAILED") >+ tc.LogStdError("{0} Compile failed with error code {1} >".format(AP_Path, ret)) >+ return 1 >+ >+ else: >+ tc.SetSuccess() >+ return 0 >diff --git a/.pytool/Plugin/CompilerPlugin/Compiler_plug_in.yaml >b/.pytool/Plugin/CompilerPlugin/Compiler_plug_in.yaml >new file mode 100644 >index 0000000000..4f9b3d3113 >--- /dev/null >+++ b/.pytool/Plugin/CompilerPlugin/Compiler_plug_in.yaml >@@ -0,0 +1,11 @@ >+## >+# CiBuildPlugin used to compile each package >+# >+# Copyright (c) 2019, Microsoft Corporation >+# SPDX-License-Identifier: BSD-2-Clause-Patent >+## >+{ >+ "scope": "cibuild", >+ "name": "Compiler Plugin", >+ "module": "CompilerPlugin" >+} >diff --git a/.pytool/Plugin/DependencyCheck/DependencyCheck.py >b/.pytool/Plugin/DependencyCheck/DependencyCheck.py >new file mode 100644 >index 0000000000..497914cf3a >--- /dev/null >+++ b/.pytool/Plugin/DependencyCheck/DependencyCheck.py >@@ -0,0 +1,120 @@ >+# @file dependency_check.py >+# >+# Copyright (c) Microsoft Corporation. All rights reserved. >+# SPDX-License-Identifier: BSD-2-Clause-Patent >+## >+ >+import logging >+import os >+from edk2toolext.environment.plugintypes.ci_build_plugin import >ICiBuildPlugin >+from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser >+from edk2toolext.environment.var_dict import VarDict >+ >+ >+class DependencyCheck(ICiBuildPlugin): >+ """ >+ A CiBuildPlugin that finds all modules (inf files) in a package and >reviews >the packages used >+ to confirm they are acceptable. This is to help enforce layering and >identify >improper >+ dependencies between packages. >+ >+ Configuration options: >+ "DependencyCheck": { >+ "AcceptableDependencies": [], # Package dec files that are allowed in >all >INFs. Example: MdePkg/MdePkg.dec >+ "AcceptableDependencies-<MODULE_TYPE>": [], # OPTIONAL Package >dependencies for INFs that are HOST_APPLICATION >+ "AcceptableDependencies-HOST_APPLICATION": [], # EXAMPLE Package >dependencies for INFs that are HOST_APPLICATION >+ "IgnoreInf": [] # Ignore INF if found in filesystem >+ } >+ """ >+ >+ def GetTestName(self, packagename: str, environment: VarDict) -> tuple: >+ """ Provide the testcase name and classname for use in reporting >+ >+ Args: >+ packagename: string containing name of package to build >+ environment: The VarDict for the test to run in >+ Returns: >+ a tuple containing the testcase name and the classname >+ (testcasename, classname) >+ testclassname: a descriptive string for the testcase can >include >whitespace >+ classname: should be patterned ><packagename>.<plugin>.<optionally any unique condition> >+ """ >+ return ("Test Package Dependencies for modules in " + packagename, >packagename + ".DependencyCheck") >+ >+ ## >+ # External function of plugin. This function is used to perform the task >of >the MuBuild Plugin >+ # >+ # - package is the edk2 path to package. This means >workspace/packagepath relative. >+ # - edk2path object configured with workspace and packages path >+ # - PkgConfig Object (dict) for the pkg >+ # - EnvConfig Object >+ # - Plugin Manager Instance >+ # - Plugin Helper Obj Instance >+ # - Junit Logger >+ # - output_stream the StringIO output stream from this plugin via >logging >+ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, >environment, PLM, PLMHelper, tc, output_stream=None): >+ overall_status = 0 >+ >+ # Get current platform >+ abs_pkg_path = >Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagen >ame) >+ >+ # Get INF Files >+ INFFiles = self.WalkDirectoryForExtension([".inf"], abs_pkg_path) >+ INFFiles = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(x) for x >in INFFiles] # make edk2relative path so can compare with Ignore List >+ >+ # Remove ignored INFs >+ if "IgnoreInf" in pkgconfig: >+ for a in pkgconfig["IgnoreInf"]: >+ a = a.replace(os.sep, "/") ## convert path sep in case >ignore list is >bad. Can't change case >+ try: >+ INFFiles.remove(a) >+ tc.LogStdOut("IgnoreInf {0}".format(a)) >+ except: >+ logging.info("DependencyConfig.IgnoreInf -> {0} not found >in >filesystem. Invalid ignore file".format(a)) >+ tc.LogStdError("DependencyConfig.IgnoreInf -> {0} not >found in >filesystem. Invalid ignore file".format(a)) >+ >+ >+ # Get the AccpetableDependencies list >+ if "AcceptableDependencies" not in pkgconfig: >+ logging.info("DependencyCheck Skipped. No Acceptable >Dependencies defined.") >+ tc.LogStdOut("DependencyCheck Skipped. No Acceptable >Dependencies defined.") >+ tc.SetSkipped() >+ return -1 >+ >+ # Log dependencies >+ for k in pkgconfig.keys(): >+ if k.startswith("AcceptableDependencies"): >+ pkgstring = "\n".join(pkgconfig[k]) >+ if ("-" in k): >+ _, _, mod_type = k.partition("-") >+ tc.LogStdOut(f"Additional dependencies for MODULE_TYPE >{mod_type}:\n {pkgstring}") >+ else: >+ tc.LogStdOut(f"Acceptable Dependencies:\n {pkgstring}") >+ >+ # For each INF file >+ for file in INFFiles: >+ ip = InfParser() >+ logging.debug("Parsing " + file) >+ >ip.SetBaseAbsPath(Edk2pathObj.WorkspacePath).SetPackagePaths(Edk2path >Obj.PackagePathList).ParseFile(file) >+ >+ if("MODULE_TYPE" not in ip.Dict): >+ tc.LogStdOut("Ignoring INF. Missing key for MODULE_TYPE >{0}".format(file)) >+ continue >+ >+ mod_type = ip.Dict["MODULE_TYPE"].upper() >+ for p in ip.PackagesUsed: >+ if p not in pkgconfig["AcceptableDependencies"]: >+ # If not in the main acceptable dependencies list then >check >module specific >+ mod_specific_key = "AcceptableDependencies-" + mod_type >+ if mod_specific_key in pkgconfig and p in >pkgconfig[mod_specific_key]: >+ continue >+ >+ logging.error("Dependency Check: Invalid Dependency INF: >{0} >depends on pkg {1}".format(file, p)) >+ tc.LogStdError("Dependency Check: Invalid Dependency INF: >{0} >depends on pkg {1}".format(file, p)) >+ overall_status += 1 >+ >+ # If XML object exists, add results >+ if overall_status is not 0: >+ tc.SetFailed("Failed with {0} errors".format(overall_status), >"DEPENDENCYCHECK_FAILED") >+ else: >+ tc.SetSuccess() >+ return overall_status >diff --git >a/.pytool/Plugin/DependencyCheck/DependencyCheck_plug_in.yaml >b/.pytool/Plugin/DependencyCheck/DependencyCheck_plug_in.yaml >new file mode 100644 >index 0000000000..fdb96d625b >--- /dev/null >+++ b/.pytool/Plugin/DependencyCheck/DependencyCheck_plug_in.yaml >@@ -0,0 +1,13 @@ >+## >+# CiBuildPlugin used to check all infs within a package >+# to confirm the packagesdependency are on the configured list of >acceptable >+# dependencies. >+# >+# Copyright (c) 2019, Microsoft Corporation >+# SPDX-License-Identifier: BSD-2-Clause-Patent >+## >+{ >+ "scope": "cibuild", >+ "name": "Dependency Check Test", >+ "module": "DependencyCheck" >+} >diff --git a/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck.py >b/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck.py >new file mode 100644 >index 0000000000..dcd8946ca6 >--- /dev/null >+++ b/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck.py >@@ -0,0 +1,118 @@ >+# @file DscCompleteCheck.py >+# >+# Copyright (c) Microsoft Corporation. All rights reserved. >+# SPDX-License-Identifier: BSD-2-Clause-Patent >+## >+import logging >+import os >+from edk2toolext.environment.plugintypes.ci_build_plugin import >ICiBuildPlugin >+from edk2toollib.uefi.edk2.parsers.dsc_parser import DscParser >+from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser >+from edk2toolext.environment.var_dict import VarDict >+ >+ >+class DscCompleteCheck(ICiBuildPlugin): >+ """ >+ A CiBuildPlugin that scans the package dsc file and confirms all modules >(inf >files) are >+ listed in the components sections. >+ >+ Configuration options: >+ "DscCompleteCheck": { >+ "DscPath": "<path to dsc from root of pkg>" >+ "IgnoreInf": [] # Ignore INF if found in filesystem by not dsc >+ } >+ """ >+ >+ def GetTestName(self, packagename: str, environment: VarDict) -> tuple: >+ """ Provide the testcase name and classname for use in reporting >+ >+ Args: >+ packagename: string containing name of package to build >+ environment: The VarDict for the test to run in >+ Returns: >+ a tuple containing the testcase name and the classname >+ (testcasename, classname) >+ testclassname: a descriptive string for the testcase can >include >whitespace >+ classname: should be patterned ><packagename>.<plugin>.<optionally any unique condition> >+ """ >+ return ("Check the " + packagename + " DSC for a being complete", >packagename + ".DscCompleteCheck") >+ >+ ## >+ # External function of plugin. This function is used to perform the task >of >the MuBuild Plugin >+ # >+ # - package is the edk2 path to package. This means >workspace/packagepath relative. >+ # - edk2path object configured with workspace and packages path >+ # - PkgConfig Object (dict) for the pkg >+ # - VarDict containing the shell environment Build Vars >+ # - Plugin Manager Instance >+ # - Plugin Helper Obj Instance >+ # - Junit Logger >+ # - output_stream the StringIO output stream from this plugin via >logging >+ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, >environment, PLM, PLMHelper, tc, output_stream=None): >+ overall_status = 0 >+ >+ # Parse the config for required DscPath element >+ if "DscPath" not in pkgconfig: >+ tc.SetSkipped() >+ tc.LogStdError("DscPath not found in config file. Nothing to >check.") >+ return -1 >+ >+ abs_pkg_path = >Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagen >ame) >+ abs_dsc_path = os.path.join(abs_pkg_path, >pkgconfig["DscPath"].strip()) >+ wsr_dsc_path = >Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(abs_dsc_path) >+ >+ if abs_dsc_path is None or wsr_dsc_path is "" or not >os.path.isfile(abs_dsc_path): >+ tc.SetSkipped() >+ tc.LogStdError("Package Dsc not found") >+ return 0 >+ >+ # Get INF Files >+ INFFiles = self.WalkDirectoryForExtension([".inf"], abs_pkg_path) >+ INFFiles = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(x) for x >in INFFiles] # make edk2relative path so can compare with DSC >+ >+ # remove ignores >+ >+ if "IgnoreInf" in pkgconfig: >+ for a in pkgconfig["IgnoreInf"]: >+ a = a.replace(os.sep, "/") >+ try: >+ tc.LogStdOut("Ignoring INF {0}".format(a)) >+ INFFiles.remove(a) >+ except: >+ tc.LogStdError("DscCompleteCheck.IgnoreInf -> {0} not >found in >filesystem. Invalid ignore file".format(a)) >+ logging.info("DscCompleteCheck.IgnoreInf -> {0} not found >in >filesystem. Invalid ignore file".format(a)) >+ >+ # DSC Parser >+ dp = DscParser() >+ dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath) >+ dp.SetPackagePaths(Edk2pathObj.PackagePathList) >+ dp.SetInputVars(environment.GetAllBuildKeyValues()) >+ dp.ParseFile(wsr_dsc_path) >+ >+ # Check if INF in component section >+ for INF in INFFiles: >+ if not any(INF.strip() in x for x in dp.ThreeMods) and \ >+ not any(INF.strip() in x for x in dp.SixMods) and \ >+ not any(INF.strip() in x for x in dp.OtherMods): >+ >+ infp = InfParser().SetBaseAbsPath(Edk2pathObj.WorkspacePath) >+ infp.SetPackagePaths(Edk2pathObj.PackagePathList) >+ infp.ParseFile(INF) >+ if("MODULE_TYPE" not in infp.Dict): >+ tc.LogStdOut("Ignoring INF. Missing key for MODULE_TYPE >{0}".format(INF)) >+ continue >+ >+ if(infp.Dict["MODULE_TYPE"] == "HOST_APPLICATION"): >+ tc.LogStdOut("Ignoring INF. Module type is >HOST_APPLICATION >{0}".format(INF)) >+ continue >+ >+ logging.critical(INF + " not in " + wsr_dsc_path) >+ tc.LogStdError("{0} not in {1}".format(INF, wsr_dsc_path)) >+ overall_status = overall_status + 1 >+ >+ # If XML object exists, add result >+ if overall_status is not 0: >+ tc.SetFailed("DscCompleteCheck {0} Failed. Errors >{1}".format(wsr_dsc_path, overall_status), "CHECK_FAILED") >+ else: >+ tc.SetSuccess() >+ return overall_status >diff --git >a/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck_plug_in.yaml >b/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck_plug_in.yaml >new file mode 100644 >index 0000000000..d84d57d973 >--- /dev/null >+++ b/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck_plug_in.yaml >@@ -0,0 +1,12 @@ >+## >+# CiBuildPlugin used to confirm all INFs are listed in >+# the components section of package dsc >+# >+# Copyright (c) 2019, Microsoft Corporation >+# SPDX-License-Identifier: BSD-2-Clause-Patent >+## >+{ >+ "scope": "cibuild", >+ "name": "Dsc Complete Check Test", >+ "module": "DscCompleteCheck" >+} >diff --git a/.pytool/Plugin/DscCompleteCheck/readme.md >b/.pytool/Plugin/DscCompleteCheck/readme.md >new file mode 100644 >index 0000000000..17e542b8d6 >--- /dev/null >+++ b/.pytool/Plugin/DscCompleteCheck/readme.md >@@ -0,0 +1,22 @@ >+# Dsc Complete Check Plugin >+ >+This CiBuildPlugin scans all INF files from a package and confirms they are >listed in the package level DSC file. The test considers it an error if any INF >does not appear in the `Components` section of the package-level DSC >(indicating that it would not be built if the package were built). This is >critical >because much of the CI infrastructure assumes that all modules will be listed >in the DSC and compiled. >+ >+## Configuration >+ >+The plugin has a few configuration options to support the UEFI codebase. >+ >+``` yaml >+"DscCompleteCheck": { >+ "DscPath": "", # Path to dsc from root of package >+ "IgnoreInf": [] # Ignore INF if found in filesystem by not dsc >+ } >+``` >+ >+### DscPath >+ >+Path to DSC to consider platform dsc >+ >+### IgnoreInf >+ >+Ignore error if Inf file is not listed in DSC file >diff --git a/.pytool/Plugin/GuidCheck/GuidCheck.py >b/.pytool/Plugin/GuidCheck/GuidCheck.py >new file mode 100644 >index 0000000000..467e17f3e8 >--- /dev/null >+++ b/.pytool/Plugin/GuidCheck/GuidCheck.py >@@ -0,0 +1,251 @@ >+# @file GuidCheck.py >+# >+# Copyright (c) Microsoft Corporation. All rights reserved. >+# SPDX-License-Identifier: BSD-2-Clause-Patent >+## >+import logging >+from edk2toolext.environment.plugintypes.ci_build_plugin import >ICiBuildPlugin >+from edk2toollib.uefi.edk2.guid_list import GuidList >+from edk2toolext.environment.var_dict import VarDict >+ >+ >+class GuidCheck(ICiBuildPlugin): >+ """ >+ A CiBuildPlugin that scans the code tree and looks for duplicate guids >+ from the package being tested. >+ >+ Configuration options: >+ "GuidCheck": { >+ "IgnoreGuidName": [], # provide in format guidname=guidvalue or just >guidname >+ "IgnoreGuidValue": [], >+ "IgnoreFoldersAndFiles": [], >+ "IgnoreDuplicates": [] # Provide in format >guidname=guidname=guidname... >+ } >+ """ >+ >+ def GetTestName(self, packagename: str, environment: VarDict) -> tuple: >+ """ Provide the testcase name and classname for use in reporting >+ >+ Args: >+ packagename: string containing name of package to build >+ environment: The VarDict for the test to run in >+ Returns: >+ a tuple containing the testcase name and the classname >+ (testcasename, classname) >+ testclassname: a descriptive string for the testcase can >include >whitespace >+ classname: should be patterned ><packagename>.<plugin>.<optionally any unique condition> >+ """ >+ return ("Confirm GUIDs are unique in " + packagename, packagename + >".GuidCheck") >+ >+ def _FindConflictingGuidValues(self, guidlist: list) -> list: >+ """ Find all duplicate guids by guid value and report them as errors >+ """ >+ # Sort the list by guid >+ guidsorted = sorted( >+ guidlist, key=lambda x: x.guid.upper(), reverse=True) >+ >+ previous = None # Store previous entry for comparison >+ error = None >+ errors = [] >+ for index in range(len(guidsorted)): >+ i = guidsorted[index] >+ if(previous is not None): >+ if i.guid == previous.guid: # Error >+ if(error is None): >+ # Catch errors with more than 1 conflict >+ error = ErrorEntry("guid") >+ error.entries.append(previous) >+ errors.append(error) >+ error.entries.append(i) >+ else: >+ # no match. clear error >+ error = None >+ previous = i >+ return errors >+ >+ def _FindConflictingGuidNames(self, guidlist: list) -> list: >+ """ Find all duplicate guids by name and if they are not all >+ from inf files report them as errors. It is ok to have >+ BASE_NAME duplication. >+ >+ Is this useful? It would catch two same named guids in dec file >+ that resolve to different values. >+ """ >+ # Sort the list by guid >+ namesorted = sorted(guidlist, key=lambda x: x.name.upper()) >+ >+ previous = None # Store previous entry for comparison >+ error = None >+ errors = [] >+ for index in range(len(namesorted)): >+ i = namesorted[index] >+ if(previous is not None): >+ # If name matches >+ if i.name == previous.name: >+ if(error is None): >+ # Catch errors with more than 1 conflict >+ error = ErrorEntry("name") >+ error.entries.append(previous) >+ errors.append(error) >+ error.entries.append(i) >+ else: >+ # no match. clear error >+ error = None >+ previous = i >+ >+ # Loop thru and remove any errors where all files are infs as it >is ok if >+ # they have the same inf base name. >+ for e in errors[:]: >+ if len( [en for en in e.entries if not >en.absfilepath.lower().endswith(".inf")]) == 0: >+ errors.remove(e) >+ >+ return errors >+ >+ ## >+ # External function of plugin. This function is used to perform the task >of >the MuBuild Plugin >+ # >+ # - package is the edk2 path to package. This means >workspace/packagepath relative. >+ # - edk2path object configured with workspace and packages path >+ # - PkgConfig Object (dict) for the pkg >+ # - EnvConfig Object >+ # - Plugin Manager Instance >+ # - Plugin Helper Obj Instance >+ # - Junit Logger >+ # - output_stream the StringIO output stream from this plugin via >logging >+ >+ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, >environment, PLM, PLMHelper, tc, output_stream=None): >+ Errors = [] >+ >+ abs_pkg_path = >Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath( >+ packagename) >+ >+ if abs_pkg_path is None: >+ tc.SetSkipped() >+ tc.LogStdError("No package {0}".format(packagename)) >+ return -1 >+ >+ All_Ignores = ["/Build", "/Conf"] >+ # Parse the config for other ignores >+ if "IgnoreFoldersAndFiles" in pkgconfig: >+ All_Ignores.extend(pkgconfig["IgnoreFoldersAndFiles"]) >+ >+ # Parse the workspace for all GUIDs >+ gs = GuidList.guidlist_from_filesystem( >+ Edk2pathObj.WorkspacePath, ignore_lines=All_Ignores) >+ >+ # Remove ignored guidvalue >+ if "IgnoreGuidValue" in pkgconfig: >+ for a in pkgconfig["IgnoreGuidValue"]: >+ try: >+ tc.LogStdOut("Ignoring Guid {0}".format(a.upper())) >+ for b in gs[:]: >+ if b.guid == a.upper(): >+ gs.remove(b) >+ except: >+ tc.LogStdError("GuidCheck.IgnoreGuid -> {0} not found. >Invalid >ignore guid".format(a.upper())) >+ logging.info("GuidCheck.IgnoreGuid -> {0} not found. >Invalid >ignore guid".format(a.upper())) >+ >+ # Remove ignored guidname >+ if "IgnoreGuidName" in pkgconfig: >+ for a in pkgconfig["IgnoreGuidName"]: >+ entry = a.split("=") >+ if(len(entry) > 2): >+ tc.LogStdError("GuidCheck.IgnoreGuidName -> {0} Invalid >Format.".format(a)) >+ logging.info("GuidCheck.IgnoreGuidName -> {0} Invalid >Format.".format(a)) >+ continue >+ try: >+ tc.LogStdOut("Ignoring Guid {0}".format(a)) >+ for b in gs[:]: >+ if b.name == entry[0]: >+ if(len(entry) == 1): >+ gs.remove(b) >+ elif(len(entry) == 2 and b.guid.upper() == >entry[1].upper()): >+ gs.remove(b) >+ else: >+ c.LogStdError("GuidCheck.IgnoreGuidName -> {0} >incomplete match. Invalid ignore guid".format(a)) >+ >+ except: >+ tc.LogStdError("GuidCheck.IgnoreGuidName -> {0} not found. >Invalid ignore name".format(a)) >+ logging.info("GuidCheck.IgnoreGuidName -> {0} not found. >Invalid >ignore name".format(a)) >+ >+ # Find conflicting Guid Values >+ Errors.extend(self._FindConflictingGuidValues(gs)) >+ >+ # Check if there are expected duplicates and remove it from the error >list >+ if "IgnoreDuplicates" in pkgconfig: >+ for a in pkgconfig["IgnoreDuplicates"]: >+ names = a.split("=") >+ if len(names) < 2: >+ tc.LogStdError("GuidCheck.IgnoreDuplicates -> {0} invalid >format".format(a)) >+ logging.info("GuidCheck.IgnoreDuplicates -> {0} invalid >format".format(a)) >+ continue >+ >+ for b in Errors[:]: >+ if b.type != "guid": >+ continue >+ ## Make a list of the names that are not in the names >list. If there >+ ## are any in the list then this error should not be >ignored. >+ t = [x for x in b.entries if x.name not in names] >+ if(len(t) == len(b.entries)): >+ ## did not apply to any entry >+ continue >+ elif(len(t) == 0): >+ ## full match - ignore duplicate >+ tc.LogStdOut("GuidCheck.IgnoreDuplicates -> >{0}".format(a)) >+ Errors.remove(b) >+ elif(len(t) < len(b.entries)): >+ ## partial match >+ tc.LogStdOut("GuidCheck.IgnoreDuplicates -> {0} >incomplete >match".format(a)) >+ logging.info("GuidCheck.IgnoreDuplicates -> {0} >incomplete >match".format(a)) >+ else: >+ tc.LogStdOut("GuidCheck.IgnoreDuplicates -> {0} >unknown >error.".format(a)) >+ logging.info("GuidCheck.IgnoreDuplicates -> {0} >unknown >error".format(a)) >+ >+ >+ >+ # Find conflicting Guid Names >+ Errors.extend(self._FindConflictingGuidNames(gs)) >+ >+ # Log errors for anything within the package under test >+ for er in Errors[:]: >+ InMyPackage = False >+ for a in er.entries: >+ if abs_pkg_path in a.absfilepath: >+ InMyPackage = True >+ break >+ if(not InMyPackage): >+ Errors.remove(er) >+ else: >+ logging.error(str(er)) >+ tc.LogStdError(str(er)) >+ >+ # add result to test case >+ overall_status = len(Errors) >+ if overall_status is not 0: >+ tc.SetFailed("GuidCheck {0} Failed. Errors {1}".format( >+ packagename, overall_status), "CHECK_FAILED") >+ else: >+ tc.SetSuccess() >+ return overall_status >+ >+ >+class ErrorEntry(): >+ """ Custom/private class for reporting errors in the GuidList >+ """ >+ >+ def __init__(self, errortype): >+ self.type = errortype # 'guid' or 'name' depending on error type >+ self.entries = [] # GuidListEntry that are in error condition >+ >+ def __str__(self): >+ a = f"Error Duplicate {self.type}: " >+ if(self.type == "guid"): >+ a += f" {self.entries[0].guid}" >+ elif(self.type == "name"): >+ a += f" {self.entries[0].name}" >+ >+ a += f" ({len(self.entries)})\n" >+ >+ for e in self.entries: >+ a += "\t" + str(e) + "\n" >+ return a >diff --git a/.pytool/Plugin/GuidCheck/GuidCheck_plug_in.yaml >b/.pytool/Plugin/GuidCheck/GuidCheck_plug_in.yaml >new file mode 100644 >index 0000000000..531efb7885 >--- /dev/null >+++ b/.pytool/Plugin/GuidCheck/GuidCheck_plug_in.yaml >@@ -0,0 +1,11 @@ >+## >+# CiBuildPlugin used to check guid uniqueness >+# >+# Copyright (c) 2019, Microsoft Corporation >+# SPDX-License-Identifier: BSD-2-Clause-Patent >+## >+{ >+ "scope": "cibuild", >+ "name": "Guid Check Test", >+ "module": "GuidCheck" >+} >diff --git a/.pytool/Plugin/GuidCheck/Readme.md >b/.pytool/Plugin/GuidCheck/Readme.md >new file mode 100644 >index 0000000000..c1bf3d728e >--- /dev/null >+++ b/.pytool/Plugin/GuidCheck/Readme.md >@@ -0,0 +1,60 @@ >+# Guid Check Plugin >+ >+This CiBuildPlugin scans all the files in a code tree to find all the GUID >definitions. After collection it will then look for duplication in the package >under test. Uniqueness of all GUIDs are critical within the UEFI environment. >Duplication can cause numerous issues including locating the wrong data >structure, calling the wrong function, or decoding the wrong data members. >+ >+Currently Scanned: >+ >+* INF files are scanned for there Module guid >+* DEC files are scanned for all of their Protocols, PPIs, and Guids as well as >the one package GUID. >+ >+Any GUID value being equal to two names or even just defined in two files is >considered an error unless in the ignore list. >+ >+Any GUID name that is found more than once is an error unless all >occurrences are Module GUIDs. Since the Module GUID is assigned to the >Module name it is common to have numerous versions of the same module >named the same. >+ >+## Configuration >+ >+The plugin has numerous configuration options to support the UEFI >codebase. >+ >+``` yaml >+"GuidCheck": { >+ "IgnoreGuidName": [], >+ "IgnoreGuidValue": [], >+ "IgnoreFoldersAndFiles": [], >+ "IgnoreDuplicates": [] >+ } >+``` >+ >+### IgnoreGuidName >+ >+This list allows strings in two formats. >+ >+* _GuidName_ >+ * This will remove any entry with this GuidName from the list of GUIDs >therefore ignoring any error associated with this name. >+* _GuidName=GuidValue_ >+ * This will also ignore the GUID by name but only if the value equals the >GuidValue. >+ * GuidValue should be in registry format. >+ * This is the suggested format to use as it will limit the ignore to only >the >defined case. >+ >+### IgnoreGuidValue >+ >+This list allows strings in guid registry format _GuidValue_. >+ >+* This will remove any entry with this GuidValue from the list of GUIDs >therefore ignoring any error associated with this value. >+* GuidValue must be in registry format xxxxxxxx-xxxx-xxxx-xxxx- >xxxxxxxxxxxx >+ >+### IgnoreFoldersAndFiles >+ >+This supports .gitignore file and folder matching strings including wildcards >+ >+* Any folder or file ignored will not be parsed and therefore any GUID >defined will be ignored. >+* The plugin will always ignores the following ["/Build", "/Conf"] >+ >+### IgnoreDuplicates >+ >+This supports strings in the format of >_GuidName_=_GuidName_=_GuidName_ >+ >+* For the error with the GuidNames to be ignored the list must match >completely with what is found during the code scan. >+ * For example if there are two GUIDs that are by design equal within the >code tree then it should be _GuidName_=_GuidName_ >+ * If instead there are three GUIDs then it must be >_GuidName_=_GuidName_=_GuidName_ >+* This is the best ignore list to use because it is the most strict and will >catch >new problems when new conflicts are introduced. >+* There are numerous places in the UEFI specification in which two GUID >names are assigned the same value. These names should be set in this ignore >list so that they don't cause an error but any additional duplication would >still >be caught. >diff --git a/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck.py >b/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck.py >new file mode 100644 >index 0000000000..33745dff11 >--- /dev/null >+++ b/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck.py >@@ -0,0 +1,153 @@ >+# @file LibraryClassCheck.py >+# >+# Copyright (c) Microsoft Corporation. All rights reserved. >+# SPDX-License-Identifier: BSD-2-Clause-Patent >+## >+import logging >+import os >+from edk2toolext.environment.plugintypes.ci_build_plugin import >ICiBuildPlugin >+from edk2toollib.uefi.edk2.parsers.dec_parser import DecParser >+from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser >+from edk2toolext.environment.var_dict import VarDict >+ >+ >+class LibraryClassCheck(ICiBuildPlugin): >+ """ >+ A CiBuildPlugin that scans the code tree and library classes for >undeclared >+ files >+ >+ Configuration options: >+ "LibraryClassCheck": { >+ IgnoreHeaderFile: [], # Ignore a file found on disk >+ IgnoreLibraryClass: [] # Ignore a declaration found in dec file >+ } >+ """ >+ >+ def GetTestName(self, packagename: str, environment: VarDict) -> tuple: >+ """ Provide the testcase name and classname for use in reporting >+ testclassname: a descriptive string for the testcase can include >whitespace >+ classname: should be patterned <packagename>.<plugin>.<optionally >any unique condition> >+ >+ Args: >+ packagename: string containing name of package to build >+ environment: The VarDict for the test to run in >+ Returns: >+ a tuple containing the testcase name and the classname >+ (testcasename, classname) >+ """ >+ return ("Check library class declarations in " + packagename, >packagename + ".LibraryClassCheck") >+ >+ def __GetPkgDec(self, rootpath): >+ try: >+ allEntries = os.listdir(rootpath) >+ for entry in allEntries: >+ if entry.lower().endswith(".dec"): >+ return(os.path.join(rootpath, entry)) >+ except Exception: >+ logging.error("Unable to find DEC for >package:{0}".format(rootpath)) >+ >+ return None >+ >+ ## >+ # External function of plugin. This function is used to perform the task >of >the MuBuild Plugin >+ # >+ # - package is the edk2 path to package. This means >workspace/packagepath relative. >+ # - edk2path object configured with workspace and packages path >+ # - PkgConfig Object (dict) for the pkg >+ # - EnvConfig Object >+ # - Plugin Manager Instance >+ # - Plugin Helper Obj Instance >+ # - Junit Logger >+ # - output_stream the StringIO output stream from this plugin via >logging >+ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, >environment, PLM, PLMHelper, tc, output_stream=None): >+ overall_status = 0 >+ LibraryClassIgnore = [] >+ >+ abs_pkg_path = >Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagen >ame) >+ abs_dec_path = self.__GetPkgDec(abs_pkg_path) >+ wsr_dec_path = >Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(abs_dec_path) >+ >+ if abs_dec_path is None or wsr_dec_path is "" or not >os.path.isfile(abs_dec_path): >+ tc.SetSkipped() >+ tc.LogStdError("No DEC file {0} in package >{1}".format(abs_dec_path, >abs_pkg_path)) >+ return -1 >+ >+ # Get all include folders >+ dec = DecParser() >+ >dec.SetBaseAbsPath(Edk2pathObj.WorkspacePath).SetPackagePaths(Edk2pa >thObj.PackagePathList) >+ dec.ParseFile(wsr_dec_path) >+ >+ AllHeaderFiles = [] >+ >+ for includepath in dec.IncludePaths: >+ ## Get all header files in the library folder >+ AbsLibraryIncludePath = os.path.join(abs_pkg_path, includepath, >"Library") >+ if(not os.path.isdir(AbsLibraryIncludePath)): >+ continue >+ >+ hfiles = self.WalkDirectoryForExtension([".h"], >AbsLibraryIncludePath) >+ hfiles = [os.path.relpath(x,abs_pkg_path) for x in hfiles] # make >package root relative path >+ hfiles = [x.replace("\\", "/") for x in hfiles] # make package >relative >path >+ >+ AllHeaderFiles.extend(hfiles) >+ >+ if len(AllHeaderFiles) == 0: >+ tc.SetSkipped() >+ tc.LogStdError(f"No Library include folder in any Include path") >+ return -1 >+ >+ # Remove ignored paths >+ if "IgnoreHeaderFile" in pkgconfig: >+ for a in pkgconfig["IgnoreHeaderFile"]: >+ try: >+ tc.LogStdOut("Ignoring Library Header File {0}".format(a)) >+ AllHeaderFiles.remove(a) >+ except: >+ tc.LogStdError("LibraryClassCheck.IgnoreHeaderFile -> {0} >not >found. Invalid Header File".format(a)) >+ logging.info("LibraryClassCheck.IgnoreHeaderFile -> {0} >not found. >Invalid Header File".format(a)) >+ >+ if "IgnoreLibraryClass" in pkgconfig: >+ LibraryClassIgnore = pkgconfig["IgnoreLibraryClass"] >+ >+ >+ ## Attempt to find library classes >+ for lcd in dec.LibraryClasses: >+ ## Check for correct file path separator >+ if "\\" in lcd.path: >+ tc.LogStdError("LibraryClassCheck.DecFilePathSeparator -> {0} >invalid.".format(lcd.path)) >+ logging.error("LibraryClassCheck.DecFilePathSeparator -> {0} >invalid.".format(lcd.path)) >+ overall_status += 1 >+ continue >+ >+ if lcd.name in LibraryClassIgnore: >+ tc.LogStdOut("Ignoring Library Class Name >{0}".format(lcd.name)) >+ LibraryClassIgnore.remove(lcd.name) >+ continue >+ >+ logging.debug(f"Looking for Library Class {lcd.path}") >+ try: >+ AllHeaderFiles.remove(lcd.path) >+ >+ except ValueError: >+ tc.LogStdError(f"Library {lcd.name} with path {lcd.path} not >found in >package filesystem") >+ logging.error(f"Library {lcd.name} with path {lcd.path} not >found in >package filesystem") >+ overall_status += 1 >+ >+ ## any remaining AllHeaderFiles are not described in DEC >+ for h in AllHeaderFiles: >+ tc.LogStdError(f"Library Header File {h} not declared in package >DEC >{wsr_dec_path}") >+ logging.error(f"Library Header File {h} not declared in package >DEC >{wsr_dec_path}") >+ overall_status += 1 >+ >+ ## Warn about any invalid library class names in the ignore list >+ for r in LibraryClassIgnore: >+ tc.LogStdError("LibraryClassCheck.IgnoreLibraryClass -> {0} not >found. >Library Class not found".format(r)) >+ logging.info("LibraryClassCheck.IgnoreLibraryClass -> {0} not >found. >Library Class not found".format(r)) >+ >+ >+ # If XML object exists, add result >+ if overall_status is not 0: >+ tc.SetFailed("LibraryClassCheck {0} Failed. Errors >{1}".format(wsr_dec_path, overall_status), "CHECK_FAILED") >+ else: >+ tc.SetSuccess() >+ return overall_status >diff --git a/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck_plug_in.yaml >b/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck_plug_in.yaml >new file mode 100644 >index 0000000000..9174453a86 >--- /dev/null >+++ b/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck_plug_in.yaml >@@ -0,0 +1,11 @@ >+## >+# CiBuildPlugin used to check that all library classes are declared correctly >in >dec file >+# >+# Copyright (c) 2019, Microsoft Corporation >+# SPDX-License-Identifier: BSD-2-Clause-Patent >+## >+{ >+ "scope": "cibuild", >+ "name": "Library Class Check Test", >+ "module": "LibraryClassCheck" >+} >diff --git a/.pytool/Plugin/LibraryClassCheck/readme.md >b/.pytool/Plugin/LibraryClassCheck/readme.md >new file mode 100644 >index 0000000000..dedee16988 >--- /dev/null >+++ b/.pytool/Plugin/LibraryClassCheck/readme.md >@@ -0,0 +1,22 @@ >+# Library Class Check Plugin >+ >+This CiBuildPlugin scans at all library header files found in the `Library` >folders >in all of the package's declared include directories and ensures that all files >have a matching LibraryClass declaration in the DEC file for the package. Any >missing declarations will cause a failure. >+ >+## Configuration >+ >+The plugin has a few configuration options to support the UEFI codebase. >+ >+``` yaml >+"LibraryClassCheck": { >+ IgnoreHeaderFile: [], # Ignore a file found on disk >+ IgnoreLibraryClass: [] # Ignore a declaration found in dec file >+} >+``` >+ >+### IgnoreHeaderFile >+ >+Ignore a file found on disk >+ >+### IgnoreLibraryClass >+ >+Ignore a declaration found in dec file >diff --git a/.pytool/Plugin/SpellCheck/Readme.md >b/.pytool/Plugin/SpellCheck/Readme.md >new file mode 100644 >index 0000000000..e0ac835191 >--- /dev/null >+++ b/.pytool/Plugin/SpellCheck/Readme.md >@@ -0,0 +1,100 @@ >+# Spell Check Plugin >+ >+This CiBuildPlugin scans all the files in a given package and checks for >spelling >errors. >+ >+This plugin requires NodeJs and cspell. If the plugin doesn't find its >required >tools then it will mark the test as skipped. >+ >+* NodeJS: https://nodejs.org/en/ >+* cspell: https://www.npmjs.com/package/cspell >+ * Src and doc available: https://github.com/streetsidesoftware/cspell >+ >+## Configuration >+ >+The plugin has a few configuration options to support the UEFI codebase. >+ >+``` yaml >+ "SpellCheck": { >+ "AuditOnly": False, # If True, log all errors and then mark as >skipped >+ "IgnoreFiles": [], # use gitignore syntax to ignore errors in >matching >files >+ "ExtendWords": [], # words to extend to the dictionary for >this >package >+ "IgnoreStandardPaths": [], # Standard Plugin defined paths that should >be ignore >+ "AdditionalIncludePaths": [] # Additional paths to spell check >(wildcards >supported) >+ } >+``` >+ >+### AuditOnly >+ >+Boolean - Default is False. >+If True run the test in an Audit only mode which will log all errors but >instead >of failing the build it will set the test as skipped. This allows visibility >into the >failures without breaking the build. >+ >+### IgnoreFiles >+ >+This supports .gitignore file and folder matching strings including wildcards >+ >+* All files will be parsed regardless but then any spelling errors found >within >ignored files will not be reported as an error. >+* Errors in ignored files will still be output to the test results as >informational >comments. >+ >+### ExtendWords >+ >+This list allows words to be added to the dictionary for the spell checker >when this package is tested. These follow the rules of the cspell config words >field. >+ >+### IgnoreStandardPaths >+ >+This plugin by default will check the below standard paths. If the package >would like to ignore any of them list that here. >+ >+```python >+[ >+# C source >+"*.c", >+"*.h", >+ >+# Assembly files >+"*.nasm", >+"*.asm", >+"*.masm", >+"*.s", >+ >+# ACPI source language >+"*.asl", >+ >+# Edk2 build files >+"*.dsc", "*.dec", "*.fdf", "*.inf", >+ >+# Documentation files >+"*.md", "*.txt" >+] >+``` >+ >+### AdditionalIncludePaths >+ >+If the package would to add additional path patterns to be included in >spellchecking they can be defined here. >+ >+## Other configuration >+ >+In the cspell.base.json there are numerous other settings configured. There >is no support to override these on a per package basis but future features >could make this available. One interesting configuration option is >`minWordLength`. Currently it is set to _5_ which means all 2,3, and 4 letter >words will be ignored. This helps minimize the number of technical acronyms, >register names, and other UEFI specific values that must be ignored . >+ >+## False positives >+ >+The cspell dictionary is not perfect and there are cases where technical >words or acronyms are not found in the dictionary. There are three ways to >resolve false positives and the choice for which method should be based on >how broadly the word should be accepted. >+ >+### CSpell Base Config file >+ >+If the change should apply to all UEFI code and documentation then it should >be added to the base config file `words` section. The base config file is >adjacent to this file and titled `cspell.base.json`. This is a list of >accepted >words for all spell checking operations on all packages. >+ >+### Package Config >+ >+In the package `*.ci.yaml` file there is a `SpellCheck` config section. This >section allows files to be ignored as well as words that should be considered >valid for all files within this package. Add the desired words to the >"ExtendedWords" member. >+ >+### In-line File >+ >+CSpell supports numerous methods to annotate your files to ignore words, >sections, etc. This can be found in CSpell documentation. Suggestion here is >to use a c-style comment at the top of the file to add words that should be >ignored just for this file. Obviously this has the highest maintenance cost >so it >should only be used for file unique words. >+ >+``` c >+// spell-checker:ignore unenroll, word2, word3 >+``` >+ >+or >+ >+```ini >+# spell-checker:ignore unenroll, word2, word3 >+``` >diff --git a/.pytool/Plugin/SpellCheck/SpellCheck.py >b/.pytool/Plugin/SpellCheck/SpellCheck.py >new file mode 100644 >index 0000000000..94ca4cd071 >--- /dev/null >+++ b/.pytool/Plugin/SpellCheck/SpellCheck.py >@@ -0,0 +1,216 @@ >+# @file SpellCheck.py >+# >+# An edk2-pytool based plugin wrapper for cspell >+# >+# Copyright (c) Microsoft Corporation. All rights reserved. >+# SPDX-License-Identifier: BSD-2-Clause-Patent >+## >+import logging >+import json >+import yaml >+from io import StringIO >+import os >+from edk2toolext.environment.plugintypes.ci_build_plugin import >ICiBuildPlugin >+from edk2toollib.utility_functions import RunCmd >+from edk2toolext.environment.var_dict import VarDict >+from edk2toollib.gitignore_parser import parse_gitignore_lines >+from edk2toolext.environment import version_aggregator >+ >+ >+class SpellCheck(ICiBuildPlugin): >+ """ >+ A CiBuildPlugin that uses the cspell node module to scan the files >+ from the package being tested for spelling errors. The plugin contains >+ the base cspell.json file then thru the configuration options other >settings >+ can be changed or extended. >+ >+ Configuration options: >+ "SpellCheck": { >+ "AuditOnly": False, # Don't fail the build if there are >errors. Just log >them >+ "IgnoreFiles": [], # use gitignore syntax to ignore errors >in matching >files >+ "ExtendWords": [], # words to extend to the dictionary for >this >package >+ "IgnoreStandardPaths": [], # Standard Plugin defined paths that >should >be ignore >+ "AdditionalIncludePaths": [] # Additional paths to spell check >(wildcards >supported) >+ } >+ """ >+ >+ # >+ # A package can remove any of these using IgnoreStandardPaths >+ # >+ STANDARD_PLUGIN_DEFINED_PATHS = ["*.c", "*.h", >+ "*.nasm", "*.asm", "*.masm", "*.s", >+ "*.asl", >+ "*.dsc", "*.dec", "*.fdf", "*.inf", >+ "*.md", "*.txt" >+ ] >+ >+ def GetTestName(self, packagename: str, environment: VarDict) -> tuple: >+ """ Provide the testcase name and classname for use in reporting >+ >+ Args: >+ packagename: string containing name of package to build >+ environment: The VarDict for the test to run in >+ Returns: >+ a tuple containing the testcase name and the classname >+ (testcasename, classname) >+ testclassname: a descriptive string for the testcase can >include >whitespace >+ classname: should be patterned ><packagename>.<plugin>.<optionally any unique condition> >+ """ >+ return ("Spell check files in " + packagename, packagename + >".SpellCheck") >+ >+ ## >+ # External function of plugin. This function is used to perform the task >of >the CiBuild Plugin >+ # >+ # - package is the edk2 path to package. This means >workspace/packagepath relative. >+ # - edk2path object configured with workspace and packages path >+ # - PkgConfig Object (dict) for the pkg >+ # - EnvConfig Object >+ # - Plugin Manager Instance >+ # - Plugin Helper Obj Instance >+ # - Junit Logger >+ # - output_stream the StringIO output stream from this plugin via >logging >+ >+ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, >environment, PLM, PLMHelper, tc, output_stream=None): >+ Errors = [] >+ >+ abs_pkg_path = >Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath( >+ packagename) >+ >+ if abs_pkg_path is None: >+ tc.SetSkipped() >+ tc.LogStdError("No package {0}".format(packagename)) >+ return -1 >+ >+ # check for node >+ return_buffer = StringIO() >+ ret = RunCmd("node", "--version", outstream=return_buffer) >+ if (ret != 0): >+ tc.SetSkipped() >+ tc.LogStdError("NodeJs not installed. Test can't run") >+ logging.warning("NodeJs not installed. Test can't run") >+ return -1 >+ node_version = return_buffer.getvalue().strip() # format vXX.XX.XX >+ tc.LogStdOut(f"Node version: {node_version}") >+ version_aggregator.GetVersionAggregator().ReportVersion( >+ "NodeJs", node_version, version_aggregator.VersionTypes.INFO) >+ >+ # Check for cspell >+ return_buffer = StringIO() >+ ret = RunCmd("cspell", "--version", outstream=return_buffer) >+ if (ret != 0): >+ tc.SetSkipped() >+ tc.LogStdError("cspell not installed. Test can't run") >+ logging.warning("cspell not installed. Test can't run") >+ return -1 >+ cspell_version = return_buffer.getvalue().strip() # format XX.XX.XX >+ tc.LogStdOut(f"CSpell version: {cspell_version}") >+ version_aggregator.GetVersionAggregator().ReportVersion( >+ "CSpell", cspell_version, version_aggregator.VersionTypes.INFO) >+ >+ package_relative_paths_to_spell_check = >SpellCheck.STANDARD_PLUGIN_DEFINED_PATHS >+ >+ # >+ # Allow the ci.yaml to remove any of the above standard paths >+ # >+ if("IgnoreStandardPaths" in pkgconfig): >+ for a in pkgconfig["IgnoreStandardPaths"]: >+ if(a in package_relative_paths_to_spell_check): >+ tc.LogStdOut( >+ f"ignoring standard path due to ci.yaml ignore: {a}") >+ package_relative_paths_to_spell_check.remove(a) >+ else: >+ tc.LogStdOut(f"Invalid IgnoreStandardPaths value: {a}") >+ >+ # >+ # check for any additional include paths defined by package config >+ # >+ if("AdditionalIncludePaths" in pkgconfig): >+ package_relative_paths_to_spell_check.extend( >+ pkgconfig["AdditionalIncludePaths"]) >+ >+ # >+ # Make the path string for cspell to check >+ # >+ relpath = os.path.relpath(abs_pkg_path) >+ cpsell_paths = " ".join( >+ [f"{relpath}/**/{x}" for x in >package_relative_paths_to_spell_check]) >+ >+ # Make the config file >+ config_file_path = os.path.join( >+ Edk2pathObj.WorkspacePath, "Build", packagename, >"cspell_actual_config.json") >+ mydir = os.path.dirname(os.path.abspath(__file__)) >+ # load as yaml so it can have comments >+ base = os.path.join(mydir, "cspell.base.yaml") >+ with open(base, "r") as i: >+ config = yaml.safe_load(i) >+ >+ if("ExtendWords" in pkgconfig): >+ config["words"].extend(pkgconfig["ExtendWords"]) >+ with open(config_file_path, "w") as o: >+ json.dump(config, o) # output as json so compat with cspell >+ >+ All_Ignores = [] >+ # Parse the config for other ignores >+ if "IgnoreFiles" in pkgconfig: >+ All_Ignores.extend(pkgconfig["IgnoreFiles"]) >+ >+ # spell check all the files >+ ignore = parse_gitignore_lines(All_Ignores, os.path.join( >+ abs_pkg_path, "nofile.txt"), abs_pkg_path) >+ >+ # result is a list of strings like this >+ # C:\src\sp-edk2\edk2\FmpDevicePkg\FmpDevicePkg.dec:53:9 - >Unknown word (Capule) >+ EasyFix = [] >+ results = self._check_spelling(cpsell_paths, config_file_path) >+ for r in results: >+ path, _, word = r.partition(" - Unknown word ") >+ if len(word) == 0: >+ # didn't find pattern >+ continue >+ >+ pathinfo = path.rsplit(":", 2) # remove the line no info >+ if(ignore(pathinfo[0])): # check against ignore list >+ tc.LogStdOut(f"ignoring error due to ci.yaml ignore: {r}") >+ continue >+ >+ # real error >+ EasyFix.append(word.strip().strip("()")) >+ Errors.append(r) >+ >+ # Log all errors tc StdError >+ for l in Errors: >+ tc.LogStdError(l.strip()) >+ >+ # Helper - Log the syntax needed to add these words to dictionary >+ if len(EasyFix) > 0: >+ EasyFix = sorted(set(a.lower() for a in EasyFix)) >+ tc.LogStdOut("\n Easy fix:") >+ OneString = "If these are not errors add this to your ci.yaml >file.\n" >+ OneString += '"SpellCheck": {\n "ExtendWords": [' >+ for a in EasyFix: >+ tc.LogStdOut(f'\n"{a}",') >+ OneString += f'\n "{a}",' >+ logging.info(OneString.rstrip(",") + '\n ]\n}') >+ >+ # add result to test case >+ overall_status = len(Errors) >+ if overall_status != 0: >+ if "AuditOnly" in pkgconfig and pkgconfig["AuditOnly"]: >+ # set as skipped if AuditOnly >+ tc.SetSkipped() >+ return -1 >+ else: >+ tc.SetFailed("SpellCheck {0} Failed. Errors {1}".format( >+ packagename, overall_status), "CHECK_FAILED") >+ else: >+ tc.SetSuccess() >+ return overall_status >+ >+ def _check_spelling(self, abs_file_to_check: str, abs_config_file_to_use: >str) -> []: >+ output = StringIO() >+ ret = RunCmd( >+ "cspell", f"--config {abs_config_file_to_use} >{abs_file_to_check}", >outstream=output) >+ if ret == 0: >+ return [] >+ else: >+ return output.getvalue().strip().splitlines() >diff --git a/.pytool/Plugin/SpellCheck/SpellCheck_plug_in.yaml >b/.pytool/Plugin/SpellCheck/SpellCheck_plug_in.yaml >new file mode 100644 >index 0000000000..161045e19e >--- /dev/null >+++ b/.pytool/Plugin/SpellCheck/SpellCheck_plug_in.yaml >@@ -0,0 +1,11 @@ >+## >+# CiBuildPlugin used to check spelling >+# >+# Copyright (c) 2019, Microsoft Corporation >+# SPDX-License-Identifier: BSD-2-Clause-Patent >+## >+{ >+ "scope": "cibuild", >+ "name": "Spell Check Test", >+ "module": "SpellCheck" >+} >diff --git a/.pytool/Plugin/SpellCheck/cspell.base.yaml >b/.pytool/Plugin/SpellCheck/cspell.base.yaml >new file mode 100644 >index 0000000000..da6c5e5da7 >--- /dev/null >+++ b/.pytool/Plugin/SpellCheck/cspell.base.yaml >@@ -0,0 +1,165 @@ >+## >+# CSpell configuration >+# >+# Copyright (c) Microsoft Corporation >+# SPDX-License-Identifier: BSD-2-Clause-Patent >+## >+{ >+ "version": "0.1", >+ "language": "en", >+ "dictionaries": [ >+ "companies ", >+ "softwareTerms", >+ "python", >+ "cpp" >+ ], >+ "ignorePaths": [ >+ "*.pdb", >+ "**/*_extdep/**", >+ "*.pdf", >+ "*.exe", >+ "*.jpg" >+ ], >+ "minWordLength": 5, >+ "allowCompoundWords": false, >+ "ignoreWords": [ >+ "muchange" >+ ], >+ "words": [ >+ "MTRRs", >+ "Microarchitecture", >+ "Goldmont", >+ "cpuid", >+ "mwait", >+ "cstate", >+ "smram", >+ "scrtm", >+ "smbus", >+ "selftest", >+ "socket", >+ "MMRAM", >+ "qword", >+ "ENDBR", >+ "SMBASE", >+ "FXSAVE", >+ "FXRSTOR", >+ "RDRAND", >+ "IOAPIC", >+ "ATAPI", >+ "movsb", >+ "iretw", >+ "XENSTORE", >+ "cdrom", >+ "oprom", >+ "oproms", >+ "varstore", >+ "EKU", >+ "ascii", >+ "nmake", >+ "NVDIMM", >+ "nasmb", >+ "Mtftp", >+ "Hypercall", >+ "hypercalls", >+ "IOMMU", >+ "QEMU", >+ "qemus", >+ "OVMF", >+ "tiano", >+ "tianocore", >+ "edkii", >+ "coreboot", >+ "uefipayload", >+ "bootloader", >+ "bootloaders", >+ "mdepkg", >+ "skuid", >+ "dxefv", >+ "toolchain", >+ "libraryclass", >+ "preboot", >+ "pythonpath", >+ "cygpath", >+ "nuget", >+ "basetools", >+ "prepi", >+ "OPTEE", >+ "stringid", >+ "peims", >+ "memmap", >+ "guids", >+ "uuids", >+ "smbios", >+ "certdb", >+ "certdbv", >+ "EfiSigList", >+ "depex", >+ "IHANDLE", >+ "Virtio", >+ "Mbytes", >+ "Citrix", >+ "initrd", >+ "semihost", >+ "Semihosting", >+ "Trustzone", >+ "Fastboot", >+ "framebuffer", >+ "genfw", >+ "TTYTERM", >+ "miniport", >+ "LFENCE", >+ "PCANSI", >+ "submodule", >+ "submodules", >+ "brotli", >+ "PCCTS", >+ "softfloat", >+ "whitepaper", >+ "ACPICA", >+ "plugfest", >+ "bringup", >+ "formset", #VFR >+ "ideqvallist", >+ "numberof", >+ "oneof", >+ "endformset", >+ "endnumeric", >+ "endoneof", >+ "disableif", >+ "guidid", >+ "classguid", >+ "efivarstore", >+ "formsetguid", >+ "formid", >+ "suppressif", >+ "grayoutif", >+ "ideqval", >+ "endform", >+ "endcheckbox", >+ "questionid", >+ "questionref", >+ "enddate", >+ "endstring", >+ "guidop", >+ "endguidop", >+ "langdef", >+ "dynamicex", >+ "tokenspace", >+ "tokenguid", >+ "pcd's", #seems like cspell bug >+ "peim's", >+ "autogen", >+ "Disasm", >+ "Torito", >+ "SRIOV", >+ "MRIOV", >+ "UARTs", >+ "Consplitter", # common module in UEFI >+ "FIFOs", >+ "ACPINVS", >+ "Endof", # due to of not being uppercase >+ "bootability", >+ "Sdhci", >+ "inmodule", >+ ] >+} >-- >2.21.0.windows.1 -=-=-=-=-=-=-=-=-=-=-=- Groups.io Links: You receive all messages sent to this group. View/Reply Online (#50175): https://edk2.groups.io/g/devel/message/50175 Mute This Topic: https://groups.io/mt/44874063/21656 Group Owner: devel+ow...@edk2.groups.io Unsubscribe: https://edk2.groups.io/g/devel/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-