From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mga01.intel.com (mga01.intel.com []) by mx.groups.io with SMTP id smtpd.web11.1594.1573461679711800722 for ; Mon, 11 Nov 2019 00:41:21 -0800 Authentication-Results: mx.groups.io; dkim=missing; spf=fail (domain: intel.com, ip: , mailfrom: michael.d.kinney@intel.com) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga101.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 11 Nov 2019 00:41:13 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.68,292,1569308400"; d="scan'208";a="234403653" Received: from mdkinney-mobl2.amr.corp.intel.com ([10.255.231.122]) by fmsmga002.fm.intel.com with ESMTP; 11 Nov 2019 00:41:13 -0800 From: "Michael D Kinney" To: devel@edk2.groups.io Cc: Sean Brogan , Bret Barkelew , Liming Gao Subject: [Patch v5 07/22] .pytool/Plugin: Add CI plugins Date: Mon, 11 Nov 2019 00:40:52 -0800 Message-Id: <20191111084107.5368-8-michael.d.kinney@intel.com> X-Mailer: git-send-email 2.21.0.windows.1 In-Reply-To: <20191111084107.5368-1-michael.d.kinney@intel.com> References: <20191111084107.5368-1-michael.d.kinney@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit From: Sean Brogan Add .pytool directory to the edk2 repository with the following plugins. These plugins are in a top level directory because that can be used with all packages and platforms. * CharEncodingCheck * CompilerPlugin * DependencyCheck * DscCompleteCheck * GuidCheck * LibraryClassCheck * SpellCheck Cc: Sean Brogan Cc: Bret Barkelew Cc: Liming Gao Signed-off-by: Michael D Kinney Reviewed-by: Liming Gao --- .../CharEncodingCheck/CharEncodingCheck.py | 118 ++++++++ .../CharEncodingCheck_plug_in.yaml | 11 + .pytool/Plugin/CharEncodingCheck/Readme.md | 18 ++ .../Plugin/CompilerPlugin/CompilerPlugin.py | 102 +++++++ .../CompilerPlugin/Compiler_plug_in.yaml | 11 + .pytool/Plugin/CompilerPlugin/Readme.md | 17 ++ .../Plugin/DependencyCheck/DependencyCheck.py | 120 +++++++++ .../DependencyCheck_plug_in.yaml | 13 + .pytool/Plugin/DependencyCheck/Readme.md | 31 +++ .../DscCompleteCheck/DscCompleteCheck.py | 118 ++++++++ .../DscCompleteCheck_plug_in.yaml | 12 + .pytool/Plugin/DscCompleteCheck/Readme.md | 27 ++ .pytool/Plugin/GuidCheck/GuidCheck.py | 251 ++++++++++++++++++ .../Plugin/GuidCheck/GuidCheck_plug_in.yaml | 11 + .pytool/Plugin/GuidCheck/Readme.md | 80 ++++++ .../LibraryClassCheck/LibraryClassCheck.py | 153 +++++++++++ .../LibraryClassCheck_plug_in.yaml | 11 + .pytool/Plugin/LibraryClassCheck/Readme.md | 25 ++ .pytool/Plugin/SpellCheck/Readme.md | 127 +++++++++ .pytool/Plugin/SpellCheck/SpellCheck.py | 216 +++++++++++++++ .../Plugin/SpellCheck/SpellCheck_plug_in.yaml | 11 + .pytool/Plugin/SpellCheck/cspell.base.yaml | 165 ++++++++++++ 22 files changed, 1648 insertions(+) create mode 100644 .pytool/Plugin/CharEncodingCheck/CharEncodingCheck.py create mode 100644 .pytool/Plugin/CharEncodingCheck/CharEncodingCheck_plug_in.yaml create mode 100644 .pytool/Plugin/CharEncodingCheck/Readme.md create mode 100644 .pytool/Plugin/CompilerPlugin/CompilerPlugin.py create mode 100644 .pytool/Plugin/CompilerPlugin/Compiler_plug_in.yaml create mode 100644 .pytool/Plugin/CompilerPlugin/Readme.md create mode 100644 .pytool/Plugin/DependencyCheck/DependencyCheck.py create mode 100644 .pytool/Plugin/DependencyCheck/DependencyCheck_plug_in.yaml create mode 100644 .pytool/Plugin/DependencyCheck/Readme.md create mode 100644 .pytool/Plugin/DscCompleteCheck/DscCompleteCheck.py create mode 100644 .pytool/Plugin/DscCompleteCheck/DscCompleteCheck_plug_in.yaml create mode 100644 .pytool/Plugin/DscCompleteCheck/Readme.md create mode 100644 .pytool/Plugin/GuidCheck/GuidCheck.py create mode 100644 .pytool/Plugin/GuidCheck/GuidCheck_plug_in.yaml create mode 100644 .pytool/Plugin/GuidCheck/Readme.md create mode 100644 .pytool/Plugin/LibraryClassCheck/LibraryClassCheck.py create mode 100644 .pytool/Plugin/LibraryClassCheck/LibraryClassCheck_plug_in.yaml create mode 100644 .pytool/Plugin/LibraryClassCheck/Readme.md create mode 100644 .pytool/Plugin/SpellCheck/Readme.md create mode 100644 .pytool/Plugin/SpellCheck/SpellCheck.py create mode 100644 .pytool/Plugin/SpellCheck/SpellCheck_plug_in.yaml create mode 100644 .pytool/Plugin/SpellCheck/cspell.base.yaml diff --git a/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck.py b/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck.py new file mode 100644 index 0000000000..54a2424875 --- /dev/null +++ b/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck.py @@ -0,0 +1,118 @@ +# @file CharEncodingCheck.py +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + + +import os +import logging +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toolext.environment.var_dict import VarDict + +## +# map +## +EcodingMap = { + ".md": 'utf-8', + ".dsc": 'utf-8', + ".dec": 'utf-8', + ".c": 'utf-8', + ".h": 'utf-8', + ".asm": 'utf-8', + ".masm": 'utf-8', + ".nasm": 'utf-8', + ".s": 'utf-8', + ".inf": 'utf-8', + ".asl": 'utf-8', + ".uni": 'utf-8', + ".py": 'utf-8' +} + + +class CharEncodingCheck(ICiBuildPlugin): + """ + A CiBuildPlugin that scans each file in the code tree and confirms the encoding is correct. + + Configuration options: + "CharEncodingCheck": { + "IgnoreFiles": [] + } + """ + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + testclassname: a descriptive string for the testcase can include whitespace + classname: should be patterned .. + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + """ + return ("Check for valid file encoding for " + packagename, packagename + ".CharEncodingCheck") + + ## + # External function of plugin. This function is used to perform the task of the ci_build_plugin Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - EnvConfig Object + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + overall_status = 0 + files_tested = 0 + + abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename) + + if abs_pkg_path is None: + tc.SetSkipped() + tc.LogStdError("No Package folder {0}".format(abs_pkg_path)) + return 0 + + for (ext, enc) in EcodingMap.items(): + files = self.WalkDirectoryForExtension([ext], abs_pkg_path) + files = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(x) for x in files] # make edk2relative path so can process ignores + + if "IgnoreFiles" in pkgconfig: + for a in pkgconfig["IgnoreFiles"]: + a = a.replace(os.sep, "/") + try: + tc.LogStdOut("Ignoring File {0}".format(a)) + files.remove(a) + except: + tc.LogStdError("CharEncodingCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a)) + logging.info("CharEncodingCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a)) + + files = [Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(x) for x in files] + for a in files: + files_tested += 1 + if(self.TestEncodingOk(a, enc)): + logging.debug("File {0} Passed Encoding Check {1}".format(a, enc)) + else: + tc.LogStdError("Encoding Failure in {0}. Not {1}".format(a, enc)) + overall_status += 1 + + tc.LogStdOut("Tested Encoding on {0} files".format(files_tested)) + if overall_status is not 0: + tc.SetFailed("CharEncoding {0} Failed. Errors {1}".format(packagename, overall_status), "CHAR_ENCODING_CHECK_FAILED") + else: + tc.SetSuccess() + return overall_status + + def TestEncodingOk(self, apath, encodingValue): + try: + with open(apath, "rb") as fobj: + fobj.read().decode(encodingValue) + except Exception as exp: + logging.error("Encoding failure: file: {0} type: {1}".format(apath, encodingValue)) + logging.debug("EXCEPTION: while processing {1} - {0}".format(exp, apath)) + return False + + return True diff --git a/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck_plug_in.yaml b/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck_plug_in.yaml new file mode 100644 index 0000000000..f0d4c70c63 --- /dev/null +++ b/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck_plug_in.yaml @@ -0,0 +1,11 @@ +## @file +# CiBuildPlugin used to check char encoding +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "cibuild", + "name": "Char Encoding Check Test", + "module": "CharEncodingCheck" +} diff --git a/.pytool/Plugin/CharEncodingCheck/Readme.md b/.pytool/Plugin/CharEncodingCheck/Readme.md new file mode 100644 index 0000000000..47486e5c3c --- /dev/null +++ b/.pytool/Plugin/CharEncodingCheck/Readme.md @@ -0,0 +1,18 @@ +# Character Encoding Check Plugin + +This CiBuildPlugin scans all the files in a package to make sure each file is +correctly encoded and all characters can be read. Improper encoding causes +tools to fail in some situations especially in different locals. + +## Configuration + +The plugin can be configured to ignore certain files. + +``` yaml +"CharEncodingCheck": { + "IgnoreFiles": [] +} +``` +### IgnoreFiles + +OPTIONAL List of file to ignore. diff --git a/.pytool/Plugin/CompilerPlugin/CompilerPlugin.py b/.pytool/Plugin/CompilerPlugin/CompilerPlugin.py new file mode 100644 index 0000000000..0a357309a4 --- /dev/null +++ b/.pytool/Plugin/CompilerPlugin/CompilerPlugin.py @@ -0,0 +1,102 @@ +# @file HostUnitTestCompiler_plugin.py +## +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +import logging +import os +import re +from edk2toollib.uefi.edk2.parsers.dsc_parser import DscParser +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toolext.environment.uefi_build import UefiBuilder +from edk2toolext import edk2_logging +from edk2toolext.environment.var_dict import VarDict + + +class CompilerPlugin(ICiBuildPlugin): + """ + A CiBuildPlugin that compiles the package dsc + from the package being tested. + + Configuration options: + "CompilerPlugin": { + "DscPath": "" + } + """ + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + """ + target = environment.GetValue("TARGET") + return ("Compile " + packagename + " " + target, packagename + ".Compiler." + target) + + def RunsOnTargetList(self): + return ["DEBUG", "RELEASE"] + + ## + # External function of plugin. This function is used to perform the task of the MuBuild Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - EnvConfig Object + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + self._env = environment + + # Parse the config for required DscPath element + if "DscPath" not in pkgconfig: + tc.SetSkipped() + tc.LogStdError("DscPath not found in config file. Nothing to compile.") + return -1 + + AP = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename) + + APDSC = os.path.join(AP, pkgconfig["DscPath"].strip()) + AP_Path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(APDSC) + if AP is None or AP_Path is None or not os.path.isfile(APDSC): + tc.SetSkipped() + tc.LogStdError("Package Dsc not found.") + return -1 + + logging.info("Building {0}".format(AP_Path)) + self._env.SetValue("ACTIVE_PLATFORM", AP_Path, "Set in Compiler Plugin") + + # Parse DSC to check for SUPPORTED_ARCHITECTURES + dp = DscParser() + dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath) + dp.SetPackagePaths(Edk2pathObj.PackagePathList) + dp.ParseFile(AP_Path) + if "SUPPORTED_ARCHITECTURES" in dp.LocalVars: + SUPPORTED_ARCHITECTURES = dp.LocalVars["SUPPORTED_ARCHITECTURES"].split('|') + TARGET_ARCHITECTURES = environment.GetValue("TARGET_ARCH").split(' ') + + # Skip if there is no intersection between SUPPORTED_ARCHITECTURES and TARGET_ARCHITECTURES + if len(set(SUPPORTED_ARCHITECTURES) & set(TARGET_ARCHITECTURES)) == 0: + tc.SetSkipped() + tc.LogStdError("No supported architecutres to build") + return -1 + + uefiBuilder = UefiBuilder() + # do all the steps + # WorkSpace, PackagesPath, PInHelper, PInManager + ret = uefiBuilder.Go(Edk2pathObj.WorkspacePath, os.pathsep.join(Edk2pathObj.PackagePathList), PLMHelper, PLM) + if ret != 0: # failure: + tc.SetFailed("Compile failed for {0}".format(packagename), "Compile_FAILED") + tc.LogStdError("{0} Compile failed with error code {1} ".format(AP_Path, ret)) + return 1 + + else: + tc.SetSuccess() + return 0 diff --git a/.pytool/Plugin/CompilerPlugin/Compiler_plug_in.yaml b/.pytool/Plugin/CompilerPlugin/Compiler_plug_in.yaml new file mode 100644 index 0000000000..3e5fb0b1df --- /dev/null +++ b/.pytool/Plugin/CompilerPlugin/Compiler_plug_in.yaml @@ -0,0 +1,11 @@ +## @file +# CiBuildPlugin used to compile each package +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "cibuild", + "name": "Compiler Plugin", + "module": "CompilerPlugin" +} diff --git a/.pytool/Plugin/CompilerPlugin/Readme.md b/.pytool/Plugin/CompilerPlugin/Readme.md new file mode 100644 index 0000000000..44e6a2c8d9 --- /dev/null +++ b/.pytool/Plugin/CompilerPlugin/Readme.md @@ -0,0 +1,17 @@ +# Compiler Plugin + +This CiBuildPlugin compiles the package DSC from the package being tested. + +## Configuration + +The package relative path of the DSC file to build. + +``` yaml +"CompilerPlugin": { + "DscPath": "" +} +``` + +### DscPath + +Package relative path to the DSC file to build. diff --git a/.pytool/Plugin/DependencyCheck/DependencyCheck.py b/.pytool/Plugin/DependencyCheck/DependencyCheck.py new file mode 100644 index 0000000000..497914cf3a --- /dev/null +++ b/.pytool/Plugin/DependencyCheck/DependencyCheck.py @@ -0,0 +1,120 @@ +# @file dependency_check.py +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +import logging +import os +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser +from edk2toolext.environment.var_dict import VarDict + + +class DependencyCheck(ICiBuildPlugin): + """ + A CiBuildPlugin that finds all modules (inf files) in a package and reviews the packages used + to confirm they are acceptable. This is to help enforce layering and identify improper + dependencies between packages. + + Configuration options: + "DependencyCheck": { + "AcceptableDependencies": [], # Package dec files that are allowed in all INFs. Example: MdePkg/MdePkg.dec + "AcceptableDependencies-": [], # OPTIONAL Package dependencies for INFs that are HOST_APPLICATION + "AcceptableDependencies-HOST_APPLICATION": [], # EXAMPLE Package dependencies for INFs that are HOST_APPLICATION + "IgnoreInf": [] # Ignore INF if found in filesystem + } + """ + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + testclassname: a descriptive string for the testcase can include whitespace + classname: should be patterned .. + """ + return ("Test Package Dependencies for modules in " + packagename, packagename + ".DependencyCheck") + + ## + # External function of plugin. This function is used to perform the task of the MuBuild Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - EnvConfig Object + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + overall_status = 0 + + # Get current platform + abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename) + + # Get INF Files + INFFiles = self.WalkDirectoryForExtension([".inf"], abs_pkg_path) + INFFiles = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(x) for x in INFFiles] # make edk2relative path so can compare with Ignore List + + # Remove ignored INFs + if "IgnoreInf" in pkgconfig: + for a in pkgconfig["IgnoreInf"]: + a = a.replace(os.sep, "/") ## convert path sep in case ignore list is bad. Can't change case + try: + INFFiles.remove(a) + tc.LogStdOut("IgnoreInf {0}".format(a)) + except: + logging.info("DependencyConfig.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a)) + tc.LogStdError("DependencyConfig.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a)) + + + # Get the AccpetableDependencies list + if "AcceptableDependencies" not in pkgconfig: + logging.info("DependencyCheck Skipped. No Acceptable Dependencies defined.") + tc.LogStdOut("DependencyCheck Skipped. No Acceptable Dependencies defined.") + tc.SetSkipped() + return -1 + + # Log dependencies + for k in pkgconfig.keys(): + if k.startswith("AcceptableDependencies"): + pkgstring = "\n".join(pkgconfig[k]) + if ("-" in k): + _, _, mod_type = k.partition("-") + tc.LogStdOut(f"Additional dependencies for MODULE_TYPE {mod_type}:\n {pkgstring}") + else: + tc.LogStdOut(f"Acceptable Dependencies:\n {pkgstring}") + + # For each INF file + for file in INFFiles: + ip = InfParser() + logging.debug("Parsing " + file) + ip.SetBaseAbsPath(Edk2pathObj.WorkspacePath).SetPackagePaths(Edk2pathObj.PackagePathList).ParseFile(file) + + if("MODULE_TYPE" not in ip.Dict): + tc.LogStdOut("Ignoring INF. Missing key for MODULE_TYPE {0}".format(file)) + continue + + mod_type = ip.Dict["MODULE_TYPE"].upper() + for p in ip.PackagesUsed: + if p not in pkgconfig["AcceptableDependencies"]: + # If not in the main acceptable dependencies list then check module specific + mod_specific_key = "AcceptableDependencies-" + mod_type + if mod_specific_key in pkgconfig and p in pkgconfig[mod_specific_key]: + continue + + logging.error("Dependency Check: Invalid Dependency INF: {0} depends on pkg {1}".format(file, p)) + tc.LogStdError("Dependency Check: Invalid Dependency INF: {0} depends on pkg {1}".format(file, p)) + overall_status += 1 + + # If XML object exists, add results + if overall_status is not 0: + tc.SetFailed("Failed with {0} errors".format(overall_status), "DEPENDENCYCHECK_FAILED") + else: + tc.SetSuccess() + return overall_status diff --git a/.pytool/Plugin/DependencyCheck/DependencyCheck_plug_in.yaml b/.pytool/Plugin/DependencyCheck/DependencyCheck_plug_in.yaml new file mode 100644 index 0000000000..121c6889cf --- /dev/null +++ b/.pytool/Plugin/DependencyCheck/DependencyCheck_plug_in.yaml @@ -0,0 +1,13 @@ +## @file +# CiBuildPlugin used to check all infs within a package +# to confirm the packagesdependency are on the configured list of acceptable +# dependencies. +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "cibuild", + "name": "Dependency Check Test", + "module": "DependencyCheck" +} diff --git a/.pytool/Plugin/DependencyCheck/Readme.md b/.pytool/Plugin/DependencyCheck/Readme.md new file mode 100644 index 0000000000..62612f6965 --- /dev/null +++ b/.pytool/Plugin/DependencyCheck/Readme.md @@ -0,0 +1,31 @@ +# Depdendency Check Plugin + +A CiBuildPlugin that finds all modules (inf files) in a package and reviews the +packages used to confirm they are acceptable. This is to help enforce layering +and identify improper dependencies between packages. + +## Configuration + +The plugin must be configured with the acceptabe package dependencies for the +package. + +``` yaml +"DependencyCheck": { + "AcceptableDependencies": [], + "AcceptableDependencies-": [], + "IgnoreInf": [] +} +``` + +### AcceptableDependencies + +Package dec files that are allowed in all INFs. Example: MdePkg/MdePkg.dec + +### AcceptableDependencies- + +OPTIONAL Package dependencies for INFs that have module type . +Example: AcceptableDependencies-HOST_APPLICATION. + +### IgnoreInf + +OPTIONAL list of INFs to ignore for this dependency check. diff --git a/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck.py b/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck.py new file mode 100644 index 0000000000..dcd8946ca6 --- /dev/null +++ b/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck.py @@ -0,0 +1,118 @@ +# @file DscCompleteCheck.py +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import logging +import os +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toollib.uefi.edk2.parsers.dsc_parser import DscParser +from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser +from edk2toolext.environment.var_dict import VarDict + + +class DscCompleteCheck(ICiBuildPlugin): + """ + A CiBuildPlugin that scans the package dsc file and confirms all modules (inf files) are + listed in the components sections. + + Configuration options: + "DscCompleteCheck": { + "DscPath": "" + "IgnoreInf": [] # Ignore INF if found in filesystem by not dsc + } + """ + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + testclassname: a descriptive string for the testcase can include whitespace + classname: should be patterned .. + """ + return ("Check the " + packagename + " DSC for a being complete", packagename + ".DscCompleteCheck") + + ## + # External function of plugin. This function is used to perform the task of the MuBuild Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - VarDict containing the shell environment Build Vars + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + overall_status = 0 + + # Parse the config for required DscPath element + if "DscPath" not in pkgconfig: + tc.SetSkipped() + tc.LogStdError("DscPath not found in config file. Nothing to check.") + return -1 + + abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename) + abs_dsc_path = os.path.join(abs_pkg_path, pkgconfig["DscPath"].strip()) + wsr_dsc_path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(abs_dsc_path) + + if abs_dsc_path is None or wsr_dsc_path is "" or not os.path.isfile(abs_dsc_path): + tc.SetSkipped() + tc.LogStdError("Package Dsc not found") + return 0 + + # Get INF Files + INFFiles = self.WalkDirectoryForExtension([".inf"], abs_pkg_path) + INFFiles = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(x) for x in INFFiles] # make edk2relative path so can compare with DSC + + # remove ignores + + if "IgnoreInf" in pkgconfig: + for a in pkgconfig["IgnoreInf"]: + a = a.replace(os.sep, "/") + try: + tc.LogStdOut("Ignoring INF {0}".format(a)) + INFFiles.remove(a) + except: + tc.LogStdError("DscCompleteCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a)) + logging.info("DscCompleteCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a)) + + # DSC Parser + dp = DscParser() + dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath) + dp.SetPackagePaths(Edk2pathObj.PackagePathList) + dp.SetInputVars(environment.GetAllBuildKeyValues()) + dp.ParseFile(wsr_dsc_path) + + # Check if INF in component section + for INF in INFFiles: + if not any(INF.strip() in x for x in dp.ThreeMods) and \ + not any(INF.strip() in x for x in dp.SixMods) and \ + not any(INF.strip() in x for x in dp.OtherMods): + + infp = InfParser().SetBaseAbsPath(Edk2pathObj.WorkspacePath) + infp.SetPackagePaths(Edk2pathObj.PackagePathList) + infp.ParseFile(INF) + if("MODULE_TYPE" not in infp.Dict): + tc.LogStdOut("Ignoring INF. Missing key for MODULE_TYPE {0}".format(INF)) + continue + + if(infp.Dict["MODULE_TYPE"] == "HOST_APPLICATION"): + tc.LogStdOut("Ignoring INF. Module type is HOST_APPLICATION {0}".format(INF)) + continue + + logging.critical(INF + " not in " + wsr_dsc_path) + tc.LogStdError("{0} not in {1}".format(INF, wsr_dsc_path)) + overall_status = overall_status + 1 + + # If XML object exists, add result + if overall_status is not 0: + tc.SetFailed("DscCompleteCheck {0} Failed. Errors {1}".format(wsr_dsc_path, overall_status), "CHECK_FAILED") + else: + tc.SetSuccess() + return overall_status diff --git a/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck_plug_in.yaml b/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck_plug_in.yaml new file mode 100644 index 0000000000..9e215d8bc6 --- /dev/null +++ b/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck_plug_in.yaml @@ -0,0 +1,12 @@ +## @file +# CiBuildPlugin used to confirm all INFs are listed in +# the components section of package dsc +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "cibuild", + "name": "Dsc Complete Check Test", + "module": "DscCompleteCheck" +} diff --git a/.pytool/Plugin/DscCompleteCheck/Readme.md b/.pytool/Plugin/DscCompleteCheck/Readme.md new file mode 100644 index 0000000000..eefbb9894d --- /dev/null +++ b/.pytool/Plugin/DscCompleteCheck/Readme.md @@ -0,0 +1,27 @@ +# Dsc Complete Check Plugin + +This CiBuildPlugin scans all INF files from a package and confirms they are +listed in the package level DSC file. The test considers it an error if any INF +does not appear in the `Components` section of the package-level DSC (indicating +that it would not be built if the package were built). This is critical because +much of the CI infrastructure assumes that all modules will be listed in the DSC +and compiled. + +## Configuration + +The plugin has a few configuration options to support the UEFI codebase. + +``` yaml +"DscCompleteCheck": { + "DscPath": "", # Path to dsc from root of package + "IgnoreInf": [] # Ignore INF if found in filesystem by not dsc + } +``` + +### DscPath + +Path to DSC to consider platform dsc + +### IgnoreInf + +Ignore error if Inf file is not listed in DSC file diff --git a/.pytool/Plugin/GuidCheck/GuidCheck.py b/.pytool/Plugin/GuidCheck/GuidCheck.py new file mode 100644 index 0000000000..467e17f3e8 --- /dev/null +++ b/.pytool/Plugin/GuidCheck/GuidCheck.py @@ -0,0 +1,251 @@ +# @file GuidCheck.py +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import logging +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toollib.uefi.edk2.guid_list import GuidList +from edk2toolext.environment.var_dict import VarDict + + +class GuidCheck(ICiBuildPlugin): + """ + A CiBuildPlugin that scans the code tree and looks for duplicate guids + from the package being tested. + + Configuration options: + "GuidCheck": { + "IgnoreGuidName": [], # provide in format guidname=guidvalue or just guidname + "IgnoreGuidValue": [], + "IgnoreFoldersAndFiles": [], + "IgnoreDuplicates": [] # Provide in format guidname=guidname=guidname... + } + """ + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + testclassname: a descriptive string for the testcase can include whitespace + classname: should be patterned .. + """ + return ("Confirm GUIDs are unique in " + packagename, packagename + ".GuidCheck") + + def _FindConflictingGuidValues(self, guidlist: list) -> list: + """ Find all duplicate guids by guid value and report them as errors + """ + # Sort the list by guid + guidsorted = sorted( + guidlist, key=lambda x: x.guid.upper(), reverse=True) + + previous = None # Store previous entry for comparison + error = None + errors = [] + for index in range(len(guidsorted)): + i = guidsorted[index] + if(previous is not None): + if i.guid == previous.guid: # Error + if(error is None): + # Catch errors with more than 1 conflict + error = ErrorEntry("guid") + error.entries.append(previous) + errors.append(error) + error.entries.append(i) + else: + # no match. clear error + error = None + previous = i + return errors + + def _FindConflictingGuidNames(self, guidlist: list) -> list: + """ Find all duplicate guids by name and if they are not all + from inf files report them as errors. It is ok to have + BASE_NAME duplication. + + Is this useful? It would catch two same named guids in dec file + that resolve to different values. + """ + # Sort the list by guid + namesorted = sorted(guidlist, key=lambda x: x.name.upper()) + + previous = None # Store previous entry for comparison + error = None + errors = [] + for index in range(len(namesorted)): + i = namesorted[index] + if(previous is not None): + # If name matches + if i.name == previous.name: + if(error is None): + # Catch errors with more than 1 conflict + error = ErrorEntry("name") + error.entries.append(previous) + errors.append(error) + error.entries.append(i) + else: + # no match. clear error + error = None + previous = i + + # Loop thru and remove any errors where all files are infs as it is ok if + # they have the same inf base name. + for e in errors[:]: + if len( [en for en in e.entries if not en.absfilepath.lower().endswith(".inf")]) == 0: + errors.remove(e) + + return errors + + ## + # External function of plugin. This function is used to perform the task of the MuBuild Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - EnvConfig Object + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + Errors = [] + + abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath( + packagename) + + if abs_pkg_path is None: + tc.SetSkipped() + tc.LogStdError("No package {0}".format(packagename)) + return -1 + + All_Ignores = ["/Build", "/Conf"] + # Parse the config for other ignores + if "IgnoreFoldersAndFiles" in pkgconfig: + All_Ignores.extend(pkgconfig["IgnoreFoldersAndFiles"]) + + # Parse the workspace for all GUIDs + gs = GuidList.guidlist_from_filesystem( + Edk2pathObj.WorkspacePath, ignore_lines=All_Ignores) + + # Remove ignored guidvalue + if "IgnoreGuidValue" in pkgconfig: + for a in pkgconfig["IgnoreGuidValue"]: + try: + tc.LogStdOut("Ignoring Guid {0}".format(a.upper())) + for b in gs[:]: + if b.guid == a.upper(): + gs.remove(b) + except: + tc.LogStdError("GuidCheck.IgnoreGuid -> {0} not found. Invalid ignore guid".format(a.upper())) + logging.info("GuidCheck.IgnoreGuid -> {0} not found. Invalid ignore guid".format(a.upper())) + + # Remove ignored guidname + if "IgnoreGuidName" in pkgconfig: + for a in pkgconfig["IgnoreGuidName"]: + entry = a.split("=") + if(len(entry) > 2): + tc.LogStdError("GuidCheck.IgnoreGuidName -> {0} Invalid Format.".format(a)) + logging.info("GuidCheck.IgnoreGuidName -> {0} Invalid Format.".format(a)) + continue + try: + tc.LogStdOut("Ignoring Guid {0}".format(a)) + for b in gs[:]: + if b.name == entry[0]: + if(len(entry) == 1): + gs.remove(b) + elif(len(entry) == 2 and b.guid.upper() == entry[1].upper()): + gs.remove(b) + else: + c.LogStdError("GuidCheck.IgnoreGuidName -> {0} incomplete match. Invalid ignore guid".format(a)) + + except: + tc.LogStdError("GuidCheck.IgnoreGuidName -> {0} not found. Invalid ignore name".format(a)) + logging.info("GuidCheck.IgnoreGuidName -> {0} not found. Invalid ignore name".format(a)) + + # Find conflicting Guid Values + Errors.extend(self._FindConflictingGuidValues(gs)) + + # Check if there are expected duplicates and remove it from the error list + if "IgnoreDuplicates" in pkgconfig: + for a in pkgconfig["IgnoreDuplicates"]: + names = a.split("=") + if len(names) < 2: + tc.LogStdError("GuidCheck.IgnoreDuplicates -> {0} invalid format".format(a)) + logging.info("GuidCheck.IgnoreDuplicates -> {0} invalid format".format(a)) + continue + + for b in Errors[:]: + if b.type != "guid": + continue + ## Make a list of the names that are not in the names list. If there + ## are any in the list then this error should not be ignored. + t = [x for x in b.entries if x.name not in names] + if(len(t) == len(b.entries)): + ## did not apply to any entry + continue + elif(len(t) == 0): + ## full match - ignore duplicate + tc.LogStdOut("GuidCheck.IgnoreDuplicates -> {0}".format(a)) + Errors.remove(b) + elif(len(t) < len(b.entries)): + ## partial match + tc.LogStdOut("GuidCheck.IgnoreDuplicates -> {0} incomplete match".format(a)) + logging.info("GuidCheck.IgnoreDuplicates -> {0} incomplete match".format(a)) + else: + tc.LogStdOut("GuidCheck.IgnoreDuplicates -> {0} unknown error.".format(a)) + logging.info("GuidCheck.IgnoreDuplicates -> {0} unknown error".format(a)) + + + + # Find conflicting Guid Names + Errors.extend(self._FindConflictingGuidNames(gs)) + + # Log errors for anything within the package under test + for er in Errors[:]: + InMyPackage = False + for a in er.entries: + if abs_pkg_path in a.absfilepath: + InMyPackage = True + break + if(not InMyPackage): + Errors.remove(er) + else: + logging.error(str(er)) + tc.LogStdError(str(er)) + + # add result to test case + overall_status = len(Errors) + if overall_status is not 0: + tc.SetFailed("GuidCheck {0} Failed. Errors {1}".format( + packagename, overall_status), "CHECK_FAILED") + else: + tc.SetSuccess() + return overall_status + + +class ErrorEntry(): + """ Custom/private class for reporting errors in the GuidList + """ + + def __init__(self, errortype): + self.type = errortype # 'guid' or 'name' depending on error type + self.entries = [] # GuidListEntry that are in error condition + + def __str__(self): + a = f"Error Duplicate {self.type}: " + if(self.type == "guid"): + a += f" {self.entries[0].guid}" + elif(self.type == "name"): + a += f" {self.entries[0].name}" + + a += f" ({len(self.entries)})\n" + + for e in self.entries: + a += "\t" + str(e) + "\n" + return a diff --git a/.pytool/Plugin/GuidCheck/GuidCheck_plug_in.yaml b/.pytool/Plugin/GuidCheck/GuidCheck_plug_in.yaml new file mode 100644 index 0000000000..d76a58a1b9 --- /dev/null +++ b/.pytool/Plugin/GuidCheck/GuidCheck_plug_in.yaml @@ -0,0 +1,11 @@ +## @file +# CiBuildPlugin used to check guid uniqueness +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "cibuild", + "name": "Guid Check Test", + "module": "GuidCheck" +} diff --git a/.pytool/Plugin/GuidCheck/Readme.md b/.pytool/Plugin/GuidCheck/Readme.md new file mode 100644 index 0000000000..4e2c353cc4 --- /dev/null +++ b/.pytool/Plugin/GuidCheck/Readme.md @@ -0,0 +1,80 @@ +# Guid Check Plugin + +This CiBuildPlugin scans all the files in a code tree to find all the GUID +definitions. After collection it will then look for duplication in the package +under test. Uniqueness of all GUIDs are critical within the UEFI environment. +Duplication can cause numerous issues including locating the wrong data +structure, calling the wrong function, or decoding the wrong data members. + +Currently Scanned: + +* INF files are scanned for there Module guid +* DEC files are scanned for all of their Protocols, PPIs, and Guids as well as + the one package GUID. + +Any GUID value being equal to two names or even just defined in two files is +considered an error unless in the ignore list. + +Any GUID name that is found more than once is an error unless all occurrences +are Module GUIDs. Since the Module GUID is assigned to the Module name it is +common to have numerous versions of the same module named the same. + +## Configuration + +The plugin has numerous configuration options to support the UEFI codebase. + +``` yaml +"GuidCheck": { + "IgnoreGuidName": [], + "IgnoreGuidValue": [], + "IgnoreFoldersAndFiles": [], + "IgnoreDuplicates": [] + } +``` + +### IgnoreGuidName + +This list allows strings in two formats. + +* _GuidName_ + * This will remove any entry with this GuidName from the list of GUIDs + therefore ignoring any error associated with this name. +* _GuidName=GuidValue_ + * This will also ignore the GUID by name but only if the value equals the + GuidValue. + * GuidValue should be in registry format. + * This is the suggested format to use as it will limit the ignore to only the + defined case. + +### IgnoreGuidValue + +This list allows strings in guid registry format _GuidValue_. + +* This will remove any entry with this GuidValue from the list of GUIDs + therefore ignoring any error associated with this value. +* GuidValue must be in registry format xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + +### IgnoreFoldersAndFiles + +This supports .gitignore file and folder matching strings including wildcards + +* Any folder or file ignored will not be parsed and therefore any GUID defined + will be ignored. +* The plugin will always ignores the following ["/Build", "/Conf"] + +### IgnoreDuplicates + +This supports strings in the format of _GuidName_=_GuidName_=_GuidName_ + +* For the error with the GuidNames to be ignored the list must match completely + with what is found during the code scan. + * For example if there are two GUIDs that are by design equal within the code + tree then it should be _GuidName_=_GuidName_ + * If instead there are three GUIDs then it must be + _GuidName_=_GuidName_=_GuidName_ +* This is the best ignore list to use because it is the most strict and will + catch new problems when new conflicts are introduced. +* There are numerous places in the UEFI specification in which two GUID names + are assigned the same value. These names should be set in this ignore list so + that they don't cause an error but any additional duplication would still be + caught. diff --git a/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck.py b/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck.py new file mode 100644 index 0000000000..33745dff11 --- /dev/null +++ b/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck.py @@ -0,0 +1,153 @@ +# @file LibraryClassCheck.py +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import logging +import os +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toollib.uefi.edk2.parsers.dec_parser import DecParser +from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser +from edk2toolext.environment.var_dict import VarDict + + +class LibraryClassCheck(ICiBuildPlugin): + """ + A CiBuildPlugin that scans the code tree and library classes for undeclared + files + + Configuration options: + "LibraryClassCheck": { + IgnoreHeaderFile: [], # Ignore a file found on disk + IgnoreLibraryClass: [] # Ignore a declaration found in dec file + } + """ + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + testclassname: a descriptive string for the testcase can include whitespace + classname: should be patterned .. + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + """ + return ("Check library class declarations in " + packagename, packagename + ".LibraryClassCheck") + + def __GetPkgDec(self, rootpath): + try: + allEntries = os.listdir(rootpath) + for entry in allEntries: + if entry.lower().endswith(".dec"): + return(os.path.join(rootpath, entry)) + except Exception: + logging.error("Unable to find DEC for package:{0}".format(rootpath)) + + return None + + ## + # External function of plugin. This function is used to perform the task of the MuBuild Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - EnvConfig Object + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + overall_status = 0 + LibraryClassIgnore = [] + + abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename) + abs_dec_path = self.__GetPkgDec(abs_pkg_path) + wsr_dec_path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(abs_dec_path) + + if abs_dec_path is None or wsr_dec_path is "" or not os.path.isfile(abs_dec_path): + tc.SetSkipped() + tc.LogStdError("No DEC file {0} in package {1}".format(abs_dec_path, abs_pkg_path)) + return -1 + + # Get all include folders + dec = DecParser() + dec.SetBaseAbsPath(Edk2pathObj.WorkspacePath).SetPackagePaths(Edk2pathObj.PackagePathList) + dec.ParseFile(wsr_dec_path) + + AllHeaderFiles = [] + + for includepath in dec.IncludePaths: + ## Get all header files in the library folder + AbsLibraryIncludePath = os.path.join(abs_pkg_path, includepath, "Library") + if(not os.path.isdir(AbsLibraryIncludePath)): + continue + + hfiles = self.WalkDirectoryForExtension([".h"], AbsLibraryIncludePath) + hfiles = [os.path.relpath(x,abs_pkg_path) for x in hfiles] # make package root relative path + hfiles = [x.replace("\\", "/") for x in hfiles] # make package relative path + + AllHeaderFiles.extend(hfiles) + + if len(AllHeaderFiles) == 0: + tc.SetSkipped() + tc.LogStdError(f"No Library include folder in any Include path") + return -1 + + # Remove ignored paths + if "IgnoreHeaderFile" in pkgconfig: + for a in pkgconfig["IgnoreHeaderFile"]: + try: + tc.LogStdOut("Ignoring Library Header File {0}".format(a)) + AllHeaderFiles.remove(a) + except: + tc.LogStdError("LibraryClassCheck.IgnoreHeaderFile -> {0} not found. Invalid Header File".format(a)) + logging.info("LibraryClassCheck.IgnoreHeaderFile -> {0} not found. Invalid Header File".format(a)) + + if "IgnoreLibraryClass" in pkgconfig: + LibraryClassIgnore = pkgconfig["IgnoreLibraryClass"] + + + ## Attempt to find library classes + for lcd in dec.LibraryClasses: + ## Check for correct file path separator + if "\\" in lcd.path: + tc.LogStdError("LibraryClassCheck.DecFilePathSeparator -> {0} invalid.".format(lcd.path)) + logging.error("LibraryClassCheck.DecFilePathSeparator -> {0} invalid.".format(lcd.path)) + overall_status += 1 + continue + + if lcd.name in LibraryClassIgnore: + tc.LogStdOut("Ignoring Library Class Name {0}".format(lcd.name)) + LibraryClassIgnore.remove(lcd.name) + continue + + logging.debug(f"Looking for Library Class {lcd.path}") + try: + AllHeaderFiles.remove(lcd.path) + + except ValueError: + tc.LogStdError(f"Library {lcd.name} with path {lcd.path} not found in package filesystem") + logging.error(f"Library {lcd.name} with path {lcd.path} not found in package filesystem") + overall_status += 1 + + ## any remaining AllHeaderFiles are not described in DEC + for h in AllHeaderFiles: + tc.LogStdError(f"Library Header File {h} not declared in package DEC {wsr_dec_path}") + logging.error(f"Library Header File {h} not declared in package DEC {wsr_dec_path}") + overall_status += 1 + + ## Warn about any invalid library class names in the ignore list + for r in LibraryClassIgnore: + tc.LogStdError("LibraryClassCheck.IgnoreLibraryClass -> {0} not found. Library Class not found".format(r)) + logging.info("LibraryClassCheck.IgnoreLibraryClass -> {0} not found. Library Class not found".format(r)) + + + # If XML object exists, add result + if overall_status is not 0: + tc.SetFailed("LibraryClassCheck {0} Failed. Errors {1}".format(wsr_dec_path, overall_status), "CHECK_FAILED") + else: + tc.SetSuccess() + return overall_status diff --git a/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck_plug_in.yaml b/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck_plug_in.yaml new file mode 100644 index 0000000000..80cb6cc118 --- /dev/null +++ b/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck_plug_in.yaml @@ -0,0 +1,11 @@ +## @file +# CiBuildPlugin used to check that all library classes are declared correctly in dec file +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "cibuild", + "name": "Library Class Check Test", + "module": "LibraryClassCheck" +} diff --git a/.pytool/Plugin/LibraryClassCheck/Readme.md b/.pytool/Plugin/LibraryClassCheck/Readme.md new file mode 100644 index 0000000000..7fa2b202f2 --- /dev/null +++ b/.pytool/Plugin/LibraryClassCheck/Readme.md @@ -0,0 +1,25 @@ +# Library Class Check Plugin + +This CiBuildPlugin scans at all library header files found in the `Library` +folders in all of the package's declared include directories and ensures that +all files have a matching LibraryClass declaration in the DEC file for the +package. Any missing declarations will cause a failure. + +## Configuration + +The plugin has a few configuration options to support the UEFI codebase. + +``` yaml +"LibraryClassCheck": { + IgnoreHeaderFile: [], # Ignore a file found on disk + IgnoreLibraryClass: [] # Ignore a declaration found in dec file +} +``` + +### IgnoreHeaderFile + +Ignore a file found on disk + +### IgnoreLibraryClass + +Ignore a declaration found in dec file diff --git a/.pytool/Plugin/SpellCheck/Readme.md b/.pytool/Plugin/SpellCheck/Readme.md new file mode 100644 index 0000000000..394bb8effc --- /dev/null +++ b/.pytool/Plugin/SpellCheck/Readme.md @@ -0,0 +1,127 @@ +# Spell Check Plugin + +This CiBuildPlugin scans all the files in a given package and checks for +spelling errors. + +This plugin requires NodeJs and cspell. If the plugin doesn't find its required +tools then it will mark the test as skipped. + +* NodeJS: https://nodejs.org/en/ +* cspell: https://www.npmjs.com/package/cspell + * Src and doc available: https://github.com/streetsidesoftware/cspell + +## Configuration + +The plugin has a few configuration options to support the UEFI codebase. + +``` yaml + "SpellCheck": { + "AuditOnly": False, # If True, log all errors and then mark as skipped + "IgnoreFiles": [], # use gitignore syntax to ignore errors in matching files + "ExtendWords": [], # words to extend to the dictionary for this package + "IgnoreStandardPaths": [], # Standard Plugin defined paths that should be ignore + "AdditionalIncludePaths": [] # Additional paths to spell check (wildcards supported) + } +``` + +### AuditOnly + +Boolean - Default is False. +If True run the test in an Audit only mode which will log all errors but instead +of failing the build it will set the test as skipped. This allows visibility +into the failures without breaking the build. + +### IgnoreFiles + +This supports .gitignore file and folder matching strings including wildcards + +* All files will be parsed regardless but then any spelling errors found within + ignored files will not be reported as an error. +* Errors in ignored files will still be output to the test results as + informational comments. + +### ExtendWords + +This list allows words to be added to the dictionary for the spell checker when +this package is tested. These follow the rules of the cspell config words field. + +### IgnoreStandardPaths + +This plugin by default will check the below standard paths. If the package +would like to ignore any of them list that here. + +```python +[ +# C source +"*.c", +"*.h", + +# Assembly files +"*.nasm", +"*.asm", +"*.masm", +"*.s", + +# ACPI source language +"*.asl", + +# Edk2 build files +"*.dsc", "*.dec", "*.fdf", "*.inf", + +# Documentation files +"*.md", "*.txt" +] +``` + +### AdditionalIncludePaths + +If the package would to add additional path patterns to be included in +spellchecking they can be defined here. + +## Other configuration + +In the cspell.base.json there are numerous other settings configured. There is +no support to override these on a per package basis but future features could +make this available. One interesting configuration option is `minWordLength`. +Currently it is set to _5_ which means all 2,3, and 4 letter words will be +ignored. This helps minimize the number of technical acronyms, register names, +and other UEFI specific values that must be ignored. + +## False positives + +The cspell dictionary is not perfect and there are cases where technical words +or acronyms are not found in the dictionary. There are three ways to resolve +false positives and the choice for which method should be based on how broadly +the word should be accepted. + +### CSpell Base Config file + +If the change should apply to all UEFI code and documentation then it should be +added to the base config file `words` section. The base config file is adjacent +to this file and titled `cspell.base.json`. This is a list of accepted words +for all spell checking operations on all packages. + +### Package Config + +In the package `*.ci.yaml` file there is a `SpellCheck` config section. This +section allows files to be ignored as well as words that should be considered +valid for all files within this package. Add the desired words to the +"ExtendedWords" member. + +### In-line File + +CSpell supports numerous methods to annotate your files to ignore words, +sections, etc. This can be found in CSpell documentation. Suggestion here is +to use a c-style comment at the top of the file to add words that should be +ignored just for this file. Obviously this has the highest maintenance cost so +it should only be used for file unique words. + +``` c +// spell-checker:ignore unenroll, word2, word3 +``` + +or + +```ini +# spell-checker:ignore unenroll, word2, word3 +``` diff --git a/.pytool/Plugin/SpellCheck/SpellCheck.py b/.pytool/Plugin/SpellCheck/SpellCheck.py new file mode 100644 index 0000000000..94ca4cd071 --- /dev/null +++ b/.pytool/Plugin/SpellCheck/SpellCheck.py @@ -0,0 +1,216 @@ +# @file SpellCheck.py +# +# An edk2-pytool based plugin wrapper for cspell +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import logging +import json +import yaml +from io import StringIO +import os +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toollib.utility_functions import RunCmd +from edk2toolext.environment.var_dict import VarDict +from edk2toollib.gitignore_parser import parse_gitignore_lines +from edk2toolext.environment import version_aggregator + + +class SpellCheck(ICiBuildPlugin): + """ + A CiBuildPlugin that uses the cspell node module to scan the files + from the package being tested for spelling errors. The plugin contains + the base cspell.json file then thru the configuration options other settings + can be changed or extended. + + Configuration options: + "SpellCheck": { + "AuditOnly": False, # Don't fail the build if there are errors. Just log them + "IgnoreFiles": [], # use gitignore syntax to ignore errors in matching files + "ExtendWords": [], # words to extend to the dictionary for this package + "IgnoreStandardPaths": [], # Standard Plugin defined paths that should be ignore + "AdditionalIncludePaths": [] # Additional paths to spell check (wildcards supported) + } + """ + + # + # A package can remove any of these using IgnoreStandardPaths + # + STANDARD_PLUGIN_DEFINED_PATHS = ["*.c", "*.h", + "*.nasm", "*.asm", "*.masm", "*.s", + "*.asl", + "*.dsc", "*.dec", "*.fdf", "*.inf", + "*.md", "*.txt" + ] + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + testclassname: a descriptive string for the testcase can include whitespace + classname: should be patterned .. + """ + return ("Spell check files in " + packagename, packagename + ".SpellCheck") + + ## + # External function of plugin. This function is used to perform the task of the CiBuild Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - EnvConfig Object + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + Errors = [] + + abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath( + packagename) + + if abs_pkg_path is None: + tc.SetSkipped() + tc.LogStdError("No package {0}".format(packagename)) + return -1 + + # check for node + return_buffer = StringIO() + ret = RunCmd("node", "--version", outstream=return_buffer) + if (ret != 0): + tc.SetSkipped() + tc.LogStdError("NodeJs not installed. Test can't run") + logging.warning("NodeJs not installed. Test can't run") + return -1 + node_version = return_buffer.getvalue().strip() # format vXX.XX.XX + tc.LogStdOut(f"Node version: {node_version}") + version_aggregator.GetVersionAggregator().ReportVersion( + "NodeJs", node_version, version_aggregator.VersionTypes.INFO) + + # Check for cspell + return_buffer = StringIO() + ret = RunCmd("cspell", "--version", outstream=return_buffer) + if (ret != 0): + tc.SetSkipped() + tc.LogStdError("cspell not installed. Test can't run") + logging.warning("cspell not installed. Test can't run") + return -1 + cspell_version = return_buffer.getvalue().strip() # format XX.XX.XX + tc.LogStdOut(f"CSpell version: {cspell_version}") + version_aggregator.GetVersionAggregator().ReportVersion( + "CSpell", cspell_version, version_aggregator.VersionTypes.INFO) + + package_relative_paths_to_spell_check = SpellCheck.STANDARD_PLUGIN_DEFINED_PATHS + + # + # Allow the ci.yaml to remove any of the above standard paths + # + if("IgnoreStandardPaths" in pkgconfig): + for a in pkgconfig["IgnoreStandardPaths"]: + if(a in package_relative_paths_to_spell_check): + tc.LogStdOut( + f"ignoring standard path due to ci.yaml ignore: {a}") + package_relative_paths_to_spell_check.remove(a) + else: + tc.LogStdOut(f"Invalid IgnoreStandardPaths value: {a}") + + # + # check for any additional include paths defined by package config + # + if("AdditionalIncludePaths" in pkgconfig): + package_relative_paths_to_spell_check.extend( + pkgconfig["AdditionalIncludePaths"]) + + # + # Make the path string for cspell to check + # + relpath = os.path.relpath(abs_pkg_path) + cpsell_paths = " ".join( + [f"{relpath}/**/{x}" for x in package_relative_paths_to_spell_check]) + + # Make the config file + config_file_path = os.path.join( + Edk2pathObj.WorkspacePath, "Build", packagename, "cspell_actual_config.json") + mydir = os.path.dirname(os.path.abspath(__file__)) + # load as yaml so it can have comments + base = os.path.join(mydir, "cspell.base.yaml") + with open(base, "r") as i: + config = yaml.safe_load(i) + + if("ExtendWords" in pkgconfig): + config["words"].extend(pkgconfig["ExtendWords"]) + with open(config_file_path, "w") as o: + json.dump(config, o) # output as json so compat with cspell + + All_Ignores = [] + # Parse the config for other ignores + if "IgnoreFiles" in pkgconfig: + All_Ignores.extend(pkgconfig["IgnoreFiles"]) + + # spell check all the files + ignore = parse_gitignore_lines(All_Ignores, os.path.join( + abs_pkg_path, "nofile.txt"), abs_pkg_path) + + # result is a list of strings like this + # C:\src\sp-edk2\edk2\FmpDevicePkg\FmpDevicePkg.dec:53:9 - Unknown word (Capule) + EasyFix = [] + results = self._check_spelling(cpsell_paths, config_file_path) + for r in results: + path, _, word = r.partition(" - Unknown word ") + if len(word) == 0: + # didn't find pattern + continue + + pathinfo = path.rsplit(":", 2) # remove the line no info + if(ignore(pathinfo[0])): # check against ignore list + tc.LogStdOut(f"ignoring error due to ci.yaml ignore: {r}") + continue + + # real error + EasyFix.append(word.strip().strip("()")) + Errors.append(r) + + # Log all errors tc StdError + for l in Errors: + tc.LogStdError(l.strip()) + + # Helper - Log the syntax needed to add these words to dictionary + if len(EasyFix) > 0: + EasyFix = sorted(set(a.lower() for a in EasyFix)) + tc.LogStdOut("\n Easy fix:") + OneString = "If these are not errors add this to your ci.yaml file.\n" + OneString += '"SpellCheck": {\n "ExtendWords": [' + for a in EasyFix: + tc.LogStdOut(f'\n"{a}",') + OneString += f'\n "{a}",' + logging.info(OneString.rstrip(",") + '\n ]\n}') + + # add result to test case + overall_status = len(Errors) + if overall_status != 0: + if "AuditOnly" in pkgconfig and pkgconfig["AuditOnly"]: + # set as skipped if AuditOnly + tc.SetSkipped() + return -1 + else: + tc.SetFailed("SpellCheck {0} Failed. Errors {1}".format( + packagename, overall_status), "CHECK_FAILED") + else: + tc.SetSuccess() + return overall_status + + def _check_spelling(self, abs_file_to_check: str, abs_config_file_to_use: str) -> []: + output = StringIO() + ret = RunCmd( + "cspell", f"--config {abs_config_file_to_use} {abs_file_to_check}", outstream=output) + if ret == 0: + return [] + else: + return output.getvalue().strip().splitlines() diff --git a/.pytool/Plugin/SpellCheck/SpellCheck_plug_in.yaml b/.pytool/Plugin/SpellCheck/SpellCheck_plug_in.yaml new file mode 100644 index 0000000000..103aff2fb1 --- /dev/null +++ b/.pytool/Plugin/SpellCheck/SpellCheck_plug_in.yaml @@ -0,0 +1,11 @@ +## @file +# CiBuildPlugin used to check spelling +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "cibuild", + "name": "Spell Check Test", + "module": "SpellCheck" +} diff --git a/.pytool/Plugin/SpellCheck/cspell.base.yaml b/.pytool/Plugin/SpellCheck/cspell.base.yaml new file mode 100644 index 0000000000..53000fc381 --- /dev/null +++ b/.pytool/Plugin/SpellCheck/cspell.base.yaml @@ -0,0 +1,165 @@ +## @file +# CSpell configuration +# +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "version": "0.1", + "language": "en", + "dictionaries": [ + "companies ", + "softwareTerms", + "python", + "cpp" + ], + "ignorePaths": [ + "*.pdb", + "**/*_extdep/**", + "*.pdf", + "*.exe", + "*.jpg" + ], + "minWordLength": 5, + "allowCompoundWords": false, + "ignoreWords": [ + "muchange" + ], + "words": [ + "MTRRs", + "Microarchitecture", + "Goldmont", + "cpuid", + "mwait", + "cstate", + "smram", + "scrtm", + "smbus", + "selftest", + "socket", + "MMRAM", + "qword", + "ENDBR", + "SMBASE", + "FXSAVE", + "FXRSTOR", + "RDRAND", + "IOAPIC", + "ATAPI", + "movsb", + "iretw", + "XENSTORE", + "cdrom", + "oprom", + "oproms", + "varstore", + "EKU", + "ascii", + "nmake", + "NVDIMM", + "nasmb", + "Mtftp", + "Hypercall", + "hypercalls", + "IOMMU", + "QEMU", + "qemus", + "OVMF", + "tiano", + "tianocore", + "edkii", + "coreboot", + "uefipayload", + "bootloader", + "bootloaders", + "mdepkg", + "skuid", + "dxefv", + "toolchain", + "libraryclass", + "preboot", + "pythonpath", + "cygpath", + "nuget", + "basetools", + "prepi", + "OPTEE", + "stringid", + "peims", + "memmap", + "guids", + "uuids", + "smbios", + "certdb", + "certdbv", + "EfiSigList", + "depex", + "IHANDLE", + "Virtio", + "Mbytes", + "Citrix", + "initrd", + "semihost", + "Semihosting", + "Trustzone", + "Fastboot", + "framebuffer", + "genfw", + "TTYTERM", + "miniport", + "LFENCE", + "PCANSI", + "submodule", + "submodules", + "brotli", + "PCCTS", + "softfloat", + "whitepaper", + "ACPICA", + "plugfest", + "bringup", + "formset", #VFR + "ideqvallist", + "numberof", + "oneof", + "endformset", + "endnumeric", + "endoneof", + "disableif", + "guidid", + "classguid", + "efivarstore", + "formsetguid", + "formid", + "suppressif", + "grayoutif", + "ideqval", + "endform", + "endcheckbox", + "questionid", + "questionref", + "enddate", + "endstring", + "guidop", + "endguidop", + "langdef", + "dynamicex", + "tokenspace", + "tokenguid", + "pcd's", #seems like cspell bug + "peim's", + "autogen", + "Disasm", + "Torito", + "SRIOV", + "MRIOV", + "UARTs", + "Consplitter", # common module in UEFI + "FIFOs", + "ACPINVS", + "Endof", # due to of not being uppercase + "bootability", + "Sdhci", + "inmodule", + ] +} -- 2.21.0.windows.1