Re: [edk2-devel] [PATCH v3 2/7] BaseTools/Plugin/CodeQL: Add CodeQL build plugin

["Yuwei Chen" <yuwei.chen@intel.com>]

Reviewed-by: Yuwei Chen <yuwei.chen@intel.com>

> -----Original Message-----
> From: mikuback@linux.microsoft.com <mikuback@linux.microsoft.com>
> Sent: Wednesday, October 18, 2023 9:05 AM
> To: devel@edk2.groups.io
> Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming
> <gaoliming@byosoft.com.cn>; Kinney, Michael D
> <michael.d.kinney@intel.com>; Rebecca Cran <rebecca@bsdio.com>; Sean
> Brogan <sean.brogan@microsoft.com>; Chen, Christine
> <yuwei.chen@intel.com>
> Subject: [PATCH v3 2/7] BaseTools/Plugin/CodeQL: Add CodeQL build plugin
> 
> From: Michael Kubacki <michael.kubacki@microsoft.com>
> 
> Adds a CodeQL plugin that supports CodeQL in the build system.
> 
> 1. CodeQlBuildPlugin - Generates a CodeQL database for a given build.
> 2. CodeQlAnalyzePlugin - Analyzes a CodeQL database and interprets
>    results.
> 3. External dependencies - Assist with downloading the CodeQL CLI and
>    making it available to the CodeQL plugins.
> 4. CodeQlQueries.qls - A C/C++ CodeQL query set run against the code.
> 5. Readme.md - A comprehensive readme file to help:
>    - Platform integrators understand how to configure the plugin
>    - Developers understand how to modify the plugin
>    - Users understand how to use the plugin
> 
> Read Readme.md for additional details.
> 
> Cc: Bob Feng <bob.c.feng@intel.com>
> Cc: Liming Gao <gaoliming@byosoft.com.cn>
> Cc: Michael D Kinney <michael.d.kinney@intel.com>
> Cc: Rebecca Cran <rebecca@bsdio.com>
> Cc: Sean Brogan <sean.brogan@microsoft.com>
> Cc: Yuwei Chen <yuwei.chen@intel.com>
> Signed-off-by: Michael Kubacki <michael.kubacki@microsoft.com>
> ---
>  BaseTools/Plugin/CodeQL/CodeQlAnalyzePlugin.py         | 222 +++++++++++
>  BaseTools/Plugin/CodeQL/CodeQlAnalyze_plug_in.yaml     |  13 +
>  BaseTools/Plugin/CodeQL/CodeQlBuildPlugin.py           | 172 +++++++++
>  BaseTools/Plugin/CodeQL/CodeQlBuild_plug_in.yaml       |  13 +
>  BaseTools/Plugin/CodeQL/CodeQlQueries.qls              |  75 ++++
>  BaseTools/Plugin/CodeQL/Readme.md                      | 388
> ++++++++++++++++++++
>  BaseTools/Plugin/CodeQL/analyze/__init__.py            |   0
>  BaseTools/Plugin/CodeQL/analyze/analyze_filter.py      | 176 +++++++++
>  BaseTools/Plugin/CodeQL/analyze/globber.py             | 132 +++++++
>  BaseTools/Plugin/CodeQL/codeqlcli_ext_dep.yaml         |  26 ++
>  BaseTools/Plugin/CodeQL/codeqlcli_linux_ext_dep.yaml   |  24 ++
>  BaseTools/Plugin/CodeQL/codeqlcli_windows_ext_dep.yaml |  24 ++
>  BaseTools/Plugin/CodeQL/common/__init__.py             |   0
>  BaseTools/Plugin/CodeQL/common/codeql_plugin.py        |  74 ++++
>  14 files changed, 1339 insertions(+)
> 
> diff --git a/BaseTools/Plugin/CodeQL/CodeQlAnalyzePlugin.py
> b/BaseTools/Plugin/CodeQL/CodeQlAnalyzePlugin.py
> new file mode 100644
> index 000000000000..199b0ad478ed
> --- /dev/null
> +++ b/BaseTools/Plugin/CodeQL/CodeQlAnalyzePlugin.py
> @@ -0,0 +1,222 @@
> +# @file CodeQAnalyzePlugin.py
> +#
> +# A build plugin that analyzes a CodeQL database.
> +#
> +# Copyright (c) Microsoft Corporation. All rights reserved.
> +# SPDX-License-Identifier: BSD-2-Clause-Patent
> +##
> +
> +import json
> +import logging
> +import os
> +import yaml
> +
> +from analyze import analyze_filter
> +from common import codeql_plugin
> +
> +from edk2toolext import edk2_logging
> +from edk2toolext.environment.plugintypes.uefi_build_plugin import \
> +    IUefiBuildPlugin
> +from edk2toolext.environment.uefi_build import UefiBuilder
> +from edk2toollib.uefi.edk2.path_utilities import Edk2Path
> +from edk2toollib.utility_functions import RunCmd
> +from pathlib import Path
> +
> +
> +class CodeQlAnalyzePlugin(IUefiBuildPlugin):
> +
> +    def do_post_build(self, builder: UefiBuilder) -> int:
> +        """CodeQL analysis post-build functionality.
> +
> +        Args:
> +            builder (UefiBuilder): A UEFI builder object for this build.
> +
> +        Returns:
> +            int: The number of CodeQL errors found. Zero indicates that
> +            AuditOnly mode is enabled or no failures were found.
> +        """
> +
> +        pp = builder.pp.split(os.pathsep)
> +        edk2_path = Edk2Path(builder.ws, pp)
> +
> +        self.builder = builder
> +        self.package = edk2_path.GetContainingPackage(
> +                            builder.mws.join(builder.ws,
> +                                             builder.env.GetValue(
> +                                                "ACTIVE_PLATFORM")))
> +        self.package_path = Path(
> +            edk2_path.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
> +                self.package))
> +        self.target = builder.env.GetValue("TARGET")
> +
> +        self.codeql_db_path = codeql_plugin.get_codeql_db_path(
> +                                builder.ws, self.package, self.target,
> +                                new_path=False)
> +
> +        self.codeql_path = codeql_plugin.get_codeql_cli_path()
> +        if not self.codeql_path:
> +            logging.critical("CodeQL build enabled but CodeQL CLI application "
> +                             "not found.")
> +            return -1
> +
> +        codeql_sarif_dir_path = self.codeql_db_path[
> +                                        :self.codeql_db_path.rindex('-')]
> +        codeql_sarif_dir_path = codeql_sarif_dir_path.replace(
> +                                        "-db-", "-analysis-")
> +        self.codeql_sarif_path = os.path.join(
> +                                        codeql_sarif_dir_path,
> +                                        (os.path.basename(
> +                                            self.codeql_db_path) +
> +                                            ".sarif"))
> +
> +        edk2_logging.log_progress(f"Analyzing {self.package} ({self.target}) "
> +                                  f"CodeQL database at:\n"
> +                                  f"           {self.codeql_db_path}")
> +        edk2_logging.log_progress(f"Results will be written to:\n"
> +                                  f"           {self.codeql_sarif_path}")
> +
> +        # Packages are allowed to specify package-specific query specifiers
> +        # in the package CI YAML file that override the global query specifier.
> +        audit_only = False
> +        query_specifiers = None
> +        package_config_file = Path(os.path.join(
> +                                self.package_path, self.package + ".ci.yaml"))
> +        plugin_data = None
> +        if package_config_file.is_file():
> +            with open(package_config_file, 'r') as cf:
> +                package_config_file_data = yaml.safe_load(cf)
> +                if "CodeQlAnalyze" in package_config_file_data:
> +                    plugin_data = package_config_file_data["CodeQlAnalyze"]
> +                    if "AuditOnly" in plugin_data:
> +                        audit_only = plugin_data["AuditOnly"]
> +                    if "QuerySpecifiers" in plugin_data:
> +                        logging.debug(f"Loading CodeQL query specifiers in "
> +                                      f"{str(package_config_file)}")
> +                        query_specifiers = plugin_data["QuerySpecifiers"]
> +
> +        global_audit_only =
> builder.env.GetValue("STUART_CODEQL_AUDIT_ONLY")
> +        if global_audit_only:
> +            if global_audit_only.strip().lower() == "true":
> +                audit_only = True
> +
> +        if audit_only:
> +            logging.info(f"CodeQL Analyze plugin is in audit only mode for "
> +                         f"{self.package} ({self.target}).")
> +
> +        # Builds can override the query specifiers defined in this plugin
> +        # by setting the value in the STUART_CODEQL_QUERY_SPECIFIERS
> +        # environment variable.
> +        if not query_specifiers:
> +            query_specifiers = builder.env.GetValue(
> +                                "STUART_CODEQL_QUERY_SPECIFIERS")
> +
> +        # Use this plugins query set file as the default fallback if it is
> +        # not overridden. It is possible the file is not present if modified
> +        # locally. In that case, skip the plugin.
> +        plugin_query_set = Path(Path(__file__).parent, "CodeQlQueries.qls")
> +
> +        if not query_specifiers and plugin_query_set.is_file():
> +            query_specifiers = str(plugin_query_set.resolve())
> +
> +        if not query_specifiers:
> +            logging.warning("Skipping CodeQL analysis since no CodeQL query "
> +                            "specifiers were provided.")
> +            return 0
> +
> +        codeql_params = (f'database analyze {self.codeql_db_path} '
> +                         f'{query_specifiers} --format=sarifv2.1.0 '
> +                         f'--output={self.codeql_sarif_path} --download '
> +                         f'--threads=0')
> +
> +        # CodeQL requires the sarif file parent directory to exist already.
> +        Path(self.codeql_sarif_path).parent.mkdir(exist_ok=True, parents=True)
> +
> +        cmd_ret = RunCmd(self.codeql_path, codeql_params)
> +        if cmd_ret != 0:
> +            logging.critical(f"CodeQL CLI analysis failed with return code "
> +                             f"{cmd_ret}.")
> +
> +        if not os.path.isfile(self.codeql_sarif_path):
> +            logging.critical(f"The sarif file {self.codeql_sarif_path} was "
> +                             f"not created. Analysis cannot continue.")
> +            return -1
> +
> +        filter_pattern_data = []
> +        global_filter_file_value = builder.env.GetValue(
> +                                    "STUART_CODEQL_FILTER_FILES")
> +        if global_filter_file_value:
> +            global_filter_files = global_filter_file_value.strip().split(',')
> +            global_filter_files = [Path(f) for f in global_filter_files]
> +
> +            for global_filter_file in global_filter_files:
> +                if global_filter_file.is_file():
> +                    with open(global_filter_file, 'r') as ff:
> +                        global_filter_file_data = yaml.safe_load(ff)
> +                        if "Filters" in global_filter_file_data:
> +                            current_pattern_data = \
> +                                global_filter_file_data["Filters"]
> +                            if type(current_pattern_data) is not list:
> +                                logging.critical(
> +                                    f"CodeQL pattern data must be a list of "
> +                                    f"strings. Data in "
> +                                    f"{str(global_filter_file.resolve())} is "
> +                                    f"invalid. CodeQL analysis is incomplete.")
> +                                return -1
> +                            filter_pattern_data += current_pattern_data
> +                        else:
> +                            logging.critical(
> +                                f"CodeQL global filter file "
> +                                f"{str(global_filter_file.resolve())} is  "
> +                                f"malformed. Missing Filters section. CodeQL "
> +                                f"analysis is incomplete.")
> +                            return -1
> +                else:
> +                    logging.critical(
> +                        f"CodeQL global filter file "
> +                        f"{str(global_filter_file.resolve())} was not found. "
> +                        f"CodeQL analysis is incomplete.")
> +                    return -1
> +
> +        if plugin_data and "Filters" in plugin_data:
> +            if type(plugin_data["Filters"]) is not list:
> +                logging.critical(
> +                    "CodeQL pattern data must be a list of strings. "
> +                    "CodeQL analysis is incomplete.")
> +                return -1
> +            filter_pattern_data.extend(plugin_data["Filters"])
> +
> +        if filter_pattern_data:
> +            logging.info("Applying CodeQL SARIF result filters.")
> +            analyze_filter.filter_sarif(
> +                self.codeql_sarif_path,
> +                self.codeql_sarif_path,
> +                filter_pattern_data,
> +                split_lines=False)
> +
> +        with open(self.codeql_sarif_path, 'r') as sf:
> +            sarif_file_data = json.load(sf)
> +
> +        try:
> +            # Perform minimal JSON parsing to find the number of errors.
> +            total_errors = 0
> +            for run in sarif_file_data['runs']:
> +                total_errors += len(run['results'])
> +        except KeyError:
> +            logging.critical("Sarif file does not contain expected data. "
> +                             "Analysis cannot continue.")
> +            return -1
> +
> +        if total_errors > 0:
> +            if audit_only:
> +                # Show a warning message so CodeQL analysis is not forgotten.
> +                # If the repo owners truly do not want to fix CodeQL issues,
> +                # analysis should be disabled entirely.
> +                logging.warning(f"{self.package} ({self.target}) CodeQL "
> +                                f"analysis ignored {total_errors} errors due "
> +                                f"to audit mode being enabled.")
> +                return 0
> +            else:
> +                logging.error(f"{self.package} ({self.target}) CodeQL "
> +                              f"analysis failed with {total_errors} errors.")
> +
> +        return total_errors
> diff --git a/BaseTools/Plugin/CodeQL/CodeQlAnalyze_plug_in.yaml
> b/BaseTools/Plugin/CodeQL/CodeQlAnalyze_plug_in.yaml
> new file mode 100644
> index 000000000000..ec01e55c53aa
> --- /dev/null
> +++ b/BaseTools/Plugin/CodeQL/CodeQlAnalyze_plug_in.yaml
> @@ -0,0 +1,13 @@
> +## @file CodeQlAnalyze_plug_in.py
> +#
> +# Build plugin used to analyze CodeQL results.
> +#
> +# Copyright (c) Microsoft Corporation. All rights reserved.
> +# SPDX-License-Identifier: BSD-2-Clause-Patent
> +##
> +
> +{
> +  "scope": "codeql-analyze",
> +  "name": "CodeQL Analyze Plugin",
> +  "module": "CodeQlAnalyzePlugin"
> +}
> diff --git a/BaseTools/Plugin/CodeQL/CodeQlBuildPlugin.py
> b/BaseTools/Plugin/CodeQL/CodeQlBuildPlugin.py
> new file mode 100644
> index 000000000000..2fbf554f8fa4
> --- /dev/null
> +++ b/BaseTools/Plugin/CodeQL/CodeQlBuildPlugin.py
> @@ -0,0 +1,172 @@
> +# @file CodeQlBuildPlugin.py
> +#
> +# A build plugin that produces CodeQL results for the present build.
> +#
> +# Copyright (c) Microsoft Corporation. All rights reserved.
> +# SPDX-License-Identifier: BSD-2-Clause-Patent
> +##
> +
> +import glob
> +import logging
> +import os
> +import stat
> +from common import codeql_plugin
> +from pathlib import Path
> +
> +from edk2toolext import edk2_logging
> +from edk2toolext.environment.plugintypes.uefi_build_plugin import \
> +    IUefiBuildPlugin
> +from edk2toolext.environment.uefi_build import UefiBuilder
> +from edk2toollib.uefi.edk2.path_utilities import Edk2Path
> +from edk2toollib.utility_functions import GetHostInfo, RemoveTree
> +
> +
> +class CodeQlBuildPlugin(IUefiBuildPlugin):
> +
> +    def do_pre_build(self, builder: UefiBuilder) -> int:
> +        """CodeQL pre-build functionality.
> +
> +        Args:
> +            builder (UefiBuilder): A UEFI builder object for this build.
> +
> +        Returns:
> +            int: The plugin return code. Zero indicates the plugin ran
> +            successfully. A non-zero value indicates an unexpected error
> +            occurred during plugin execution.
> +        """
> +
> +        if not builder.SkipBuild:
> +            pp = builder.pp.split(os.pathsep)
> +            edk2_path = Edk2Path(builder.ws, pp)
> +
> +            self.builder = builder
> +            self.package = edk2_path.GetContainingPackage(
> +                                builder.mws.join(builder.ws,
> +                                                builder.env.GetValue(
> +                                                    "ACTIVE_PLATFORM")))
> +            self.target = builder.env.GetValue("TARGET")
> +
> +            self.build_output_dir = builder.env.GetValue("BUILD_OUTPUT_BASE")
> +
> +            self.codeql_db_path = codeql_plugin.get_codeql_db_path(
> +                                    builder.ws, self.package, self.target)
> +
> +            edk2_logging.log_progress(f"{self.package} will be built for CodeQL")
> +            edk2_logging.log_progress(f"  CodeQL database will be written to "
> +                                    f"{self.codeql_db_path}")
> +
> +            self.codeql_path = codeql_plugin.get_codeql_cli_path()
> +            if not self.codeql_path:
> +                logging.critical("CodeQL build enabled but CodeQL CLI application "
> +                                "not found.")
> +                return -1
> +
> +            # CodeQL can only generate a database on clean build
> +            #
> +            # Note: builder.CleanTree() cannot be used here as some platforms
> +            #       have build steps that run before this plugin that store
> +            #       files in the build output directory.
> +            #
> +            #       CodeQL does not care about with those files or many others
> such
> +            #       as the FV directory, build logs, etc. so instead focus on
> +            #       removing only the directories with compilation/linker output
> +            #       for the architectures being built (that need clean runs for
> +            #       CodeQL to work).
> +            targets = self.builder.env.GetValue("TARGET_ARCH").split(" ")
> +            for target in targets:
> +                directory_to_delete = Path(self.build_output_dir, target)
> +
> +                if directory_to_delete.is_dir():
> +                    logging.debug(f"Removing {str(directory_to_delete)} to have a "
> +                                f"clean build for CodeQL.")
> +                    RemoveTree(str(directory_to_delete))
> +
> +            # CodeQL CLI does not handle spaces passed in CLI commands well
> +            # (perhaps at all) as discussed here:
> +            #   1. https://github.com/github/codeql-cli-binaries/issues/73
> +            #   2. https://github.com/github/codeql/issues/4910
> +            #
> +            # Since it's unclear how quotes are handled and may change in the
> +            # future, this code is going to use the workaround to place the
> +            # command in an executable file that is instead passed to CodeQL.
> +            self.codeql_cmd_path = Path(builder.mws.join(
> +                                        builder.ws, self.build_output_dir,
> +                                        "codeql_build_command"))
> +
> +            build_params = self._get_build_params()
> +
> +            codeql_build_cmd = ""
> +            if GetHostInfo().os == "Windows":
> +                self.codeql_cmd_path = self.codeql_cmd_path.parent / (
> +                    self.codeql_cmd_path.name + '.bat')
> +            elif GetHostInfo().os == "Linux":
> +                self.codeql_cmd_path = self.codeql_cmd_path.parent / (
> +                    self.codeql_cmd_path.name + '.sh')
> +                codeql_build_cmd += f"#!/bin/bash{os.linesep * 2}"
> +            codeql_build_cmd += "build " + build_params
> +
> +            self.codeql_cmd_path.parent.mkdir(exist_ok=True, parents=True)
> +            self.codeql_cmd_path.write_text(encoding='utf8',
> data=codeql_build_cmd)
> +
> +            if GetHostInfo().os == "Linux":
> +                os.chmod(self.codeql_cmd_path,
> +                        os.stat(self.codeql_cmd_path).st_mode | stat.S_IEXEC)
> +                for f in glob.glob(os.path.join(
> +                    os.path.dirname(self.codeql_path), '**/*'), recursive=True):
> +                        os.chmod(f, os.stat(f).st_mode | stat.S_IEXEC)
> +
> +            codeql_params = (f'database create {self.codeql_db_path} '
> +                            f'--language=cpp '
> +                            f'--source-root={builder.ws} '
> +                            f'--command={self.codeql_cmd_path}')
> +
> +            # Set environment variables so the CodeQL build command is picked
> up
> +            # as the active build command.
> +            #
> +            # Note: Requires recent changes in edk2-pytool-extensions (0.20.0)
> +            #       to support reading these variables.
> +            builder.env.SetValue(
> +                "EDK_BUILD_CMD", self.codeql_path, "Set in CodeQL Build Plugin")
> +            builder.env.SetValue(
> +                "EDK_BUILD_PARAMS", codeql_params, "Set in CodeQL Build
> Plugin")
> +
> +        return 0
> +
> +    def _get_build_params(self) -> str:
> +        """Returns the build command parameters for this build.
> +
> +        Based on the well-defined `build` command-line parameters.
> +
> +        Returns:
> +            str: A string representing the parameters for the build command.
> +        """
> +        build_params = f"-p {self.builder.env.GetValue('ACTIVE_PLATFORM')}"
> +        build_params += f" -b {self.target}"
> +        build_params += f" -t {self.builder.env.GetValue('TOOL_CHAIN_TAG')}"
> +
> +        max_threads =
> self.builder.env.GetValue('MAX_CONCURRENT_THREAD_NUMBER')
> +        if max_threads is not None:
> +            build_params += f" -n {max_threads}"
> +
> +        rt = self.builder.env.GetValue("TARGET_ARCH").split(" ")
> +        for t in rt:
> +            build_params += " -a " + t
> +
> +        if (self.builder.env.GetValue("BUILDREPORTING") == "TRUE"):
> +            build_params += (" -y " +
> +                             self.builder.env.GetValue("BUILDREPORT_FILE"))
> +            rt = self.builder.env.GetValue("BUILDREPORT_TYPES").split(" ")
> +            for t in rt:
> +                build_params += " -Y " + t
> +
> +        # add special processing to handle building a single module
> +        mod = self.builder.env.GetValue("BUILDMODULE")
> +        if (mod is not None and len(mod.strip()) > 0):
> +            build_params += " -m " + mod
> +            edk2_logging.log_progress("Single Module Build: " + mod)
> +
> +        build_vars = self.builder.env.GetAllBuildKeyValues(self.target)
> +        for key, value in build_vars.items():
> +            build_params += " -D " + key + "=" + value
> +
> +        return build_params
> diff --git a/BaseTools/Plugin/CodeQL/CodeQlBuild_plug_in.yaml
> b/BaseTools/Plugin/CodeQL/CodeQlBuild_plug_in.yaml
> new file mode 100644
> index 000000000000..13baa58d0cdf
> --- /dev/null
> +++ b/BaseTools/Plugin/CodeQL/CodeQlBuild_plug_in.yaml
> @@ -0,0 +1,13 @@
> +## @file CodeQlBuild_plug_in.py
> +#
> +# Build plugin used to produce a CodeQL database from a build.
> +#
> +# Copyright (c) Microsoft Corporation. All rights reserved.
> +# SPDX-License-Identifier: BSD-2-Clause-Patent
> +##
> +
> +{
> +  "scope": "codeql-build",
> +  "name": "CodeQL Build Plugin",
> +  "module": "CodeQlBuildPlugin"
> +}
> diff --git a/BaseTools/Plugin/CodeQL/CodeQlQueries.qls
> b/BaseTools/Plugin/CodeQL/CodeQlQueries.qls
> new file mode 100644
> index 000000000000..3f97bcd583d5
> --- /dev/null
> +++ b/BaseTools/Plugin/CodeQL/CodeQlQueries.qls
> @@ -0,0 +1,75 @@
> +---
> +- description: C++ queries
> +
> +- queries: '.'
> +  from: codeql/cpp-queries
> +
> +###############################################################
> ###########################
> +# Queries
> +###############################################################
> ###########################
> +
> +## Enable When Time is Available to Fix Issues
> +# Hundreds of issues. Most appear valid. Type: Recommendation.
> +#- include:
> +#    id: cpp/missing-null-test
> +
> +## Errors
> +- include:
> +    id: cpp/overrunning-write
> +- include:
> +    id: cpp/overrunning-write-with-float
> +- include:
> +    id: cpp/pointer-overflow-check
> +- include:
> +    id: cpp/very-likely-overrunning-write
> +
> +## Warnings
> +- include:
> +    id: cpp/conditionallyuninitializedvariable
> +- include:
> +    id: cpp/infinite-loop-with-unsatisfiable-exit-condition
> +- include:
> +    id: cpp/overflow-buffer
> +
> +# Note: Some queries above are not active by default with the below filter.
> +#       Update the filter and run the queries again to get all results.
> +- include:
> +    tags:
> +      - "security"
> +      - "correctness"
> +    severity:
> +      - "error"
> +      - "warning"
> +      - "recommendation"
> +
> +# Specifically hide the results of these.
> +#
> +# The following rules have been evaluated and explicitly not included for the
> following reasons:
> +#   - `cpp/allocation-too-small` - Appears to be hardcoded for C standard
> library functions `malloc`, `calloc`,
> +#     `realloc`, so it consumes time without much value with custom allocation
> functions in the codebase.
> +#   - `cpp/commented-out-code` - Triggers often. Needs further review.
> +#   - `cpp/duplicate-include-guard` - The <Phase>EntryPoint.h files includes a
> common include guard value
> +#     `__MODULE_ENTRY_POINT_H__`. This was the only occurrence found.
> So not very useful.
> +#   - `cpp/invalid-pointer-deref` - Very limited results with what appear to be
> false positives.
> +#   - `cpp/use-of-goto` - Goto is valid and allowed in the codebase.
> +#   - `cpp/useless-expression` - Triggers too often on cases where a NULL lib
> implementation is provided for a function.
> +#     Because the implementation simply returns, the check considers it
> useless.
> +#   - `cpp/weak-crypto/*` - Crypto algorithms are tracked outside CodeQL.
> +- exclude:
> +    id: cpp/allocation-too-small
> +- exclude:
> +    id: cpp/commented-out-code
> +- exclude:
> +    id: cpp/duplicate-include-guard
> +- exclude:
> +    id: cpp/invalid-pointer-deref
> +- exclude:
> +    id: cpp/use-of-goto
> +- exclude:
> +    id: cpp/useless-expression
> +- exclude:
> +    id: cpp/weak-crypto/banned-hash-algorithms
> +- exclude:
> +    id: cpp/weak-crypto/capi/banned-modes
> +- exclude:
> +    id: cpp/weak-crypto/openssl/banned-hash-algorithms
> diff --git a/BaseTools/Plugin/CodeQL/Readme.md
> b/BaseTools/Plugin/CodeQL/Readme.md
> new file mode 100644
> index 000000000000..18587e2b258d
> --- /dev/null
> +++ b/BaseTools/Plugin/CodeQL/Readme.md
> @@ -0,0 +1,388 @@
> +# CodeQL Plugin
> +
> +The set of CodeQL plugins provided include two main plugins that
> seamlessly integrate into a Stuart build environment:
> +
> +1. `CodeQlBuildPlugin` - Used to produce a CodeQL database from a build.
> +2. `CodeQlAnalyzePlugin` - Used to analyze a CodeQL database.
> +
> +While CodeQL can be run in a CI environment with other approaches. This
> plugin offers the following advantages:
> +
> +1. Provides exactly the same results locally as on a CI server.
> +2. Integrates very well into VS Code.
> +3. Very simple to use - just use normal Stuart update and build commands.
> +4. Very simple to understand - minimally wraps the official CodeQL CLI.
> +5. Very simple to integrate - works like any other Stuart build plugin.
> +   - Integration is usually just a few lines of code.
> +6. Portable - not tied to Azure DevOps specific, GitHub specific, or other host
> infrastructure.
> +7. Versioned - the query and filters are versioned in source control so easy to
> find and track.
> +
> +It is very important to read the Integration Instructions in this file and
> determine how to best integrate the
> +CodeQL plugin into your environment.
> +
> +Due to the total size of dependencies required to run CodeQL and the
> flexibility needed by a platform to determine what
> +CodeQL queries to run and how to interpret results, a number of
> configuration options are provided to allow a high
> +degree of flexibility during platform integration.
> +
> +This document is focused on those setting up the CodeQL plugin in their
> environment. Once setup, end users simply need
> +to use their normal build commands and process and CodeQL will be
> integrated with it. The most relevant section for
> +such users is [Local Development Tips](#local-development-tips).
> +
> +## Table of Contents
> +
> +1. [Database and Analysis Result Locations](#database-and-analysis-result-
> locations)
> +2. [Global Configuration](#global-configuration)
> +3. [Package-Specific Configuration](#package-specific-configuration)
> +4. [Filter Patterns](#filter-patterns)
> +5. [Integration Instructions](#integration-instructions)
> +   - [Integration Step 1 - Choose Scopes](#integration-step-1---choose-scopes)
> +     - [Scopes Available](#scopes-available)
> +   - [Integration Step 2 - Choose CodeQL Queries](#integration-step-2---
> choose-codeql-queries)
> +   - [Integration Step 3 - Determine Global Configuration Values](#integration-
> step-3---determine-global-configuration-values)
> +   - [Integration Step 4 - Determine Package-Specific Configuration
> Values](#integration-step-4---determine-package-specific-configuration-
> values)
> +   - [Integration Step 5 - Testing](#integration-step-5---testing)
> +   - [Integration Step 6 - Define Inclusion and Exclusion Filter
> Patterns](#integration-step-6---define-inclusion-and-exclusion-filter-patterns)
> +6. [High-Level Operation](#high-level-operation)
> +   - [CodeQlBuildPlugin](#codeqlbuildplugin)
> +   - [CodeQlAnalyzePlugin](#codeqlanalyzeplugin)
> +7. [Local Development Tips](#local-development-tips)
> +8. [Resolution Guidelines](#resolution-guidelines)
> +
> +## Database and Analysis Result Locations
> +
> +The CodeQL database is written to a directory unique to the package and
> target being built:
> +
> +  `Build/codeql-db-<package>-<target>-<instance>`
> +
> +For example: `Build/codeql-db-mdemodulepkg-debug-0`
> +
> +The plugin does not delete or overwrite existing databases, the instance
> value is simply increased. This is
> +because databases are large, take a long time to generate, and are important
> for reproducing analysis results. The user
> +is responsible for deleting database directories when they are no longer
> needed.
> +
> +Similarly, analysis results are written to a directory unique to the package
> and target. For analysis, results are
> +stored in individual files so those files are stored in a single directory.
> +
> +For example, all analysis results for the above package and target will be
> stored in:
> +  `codeql-analysis-mdemodulepkg-debug`
> +
> +CodeQL results are stored in [SARIF](https://sarifweb.azurewebsites.net/)
> (Static Analysis Results Interchange Format)
> +([CodeQL SARIF documentation](https://codeql.github.com/docs/codeql-
> cli/sarif-output/)) files. Each SARIF file
> +corresponding to a database will be stored in a file with an instance
> matching the database instance.
> +
> +For example, the analysis result file for the above database would be stored
> in this file:
> +  `codeql-analysis-mdemodulepkg-debug/codeql-db-mdemodulepkg-debug-
> 0.sarif`
> +
> +Result files are overwritten. This is because result files are quick to generate
> and need to represent the latest
> +results for the last analysis operation performed. The user is responsible for
> backing up SARIF result files if they
> +need to saved.
> +
> +## Global Configuration
> +
> +Global configuration values are specified with build environment variables.
> +
> +These values are all optional. They provide a convenient mechanism for a
> build script to set the value for all packages
> +built by the script.
> +
> +- `STUART_CODEQL_AUDIT_ONLY` - If `true` (case insensitive),
> `CodeQlAnalyzePlugin` will be in audit-only mode. In this
> +  mode all CodeQL failures are ignored.
> +- `STUART_CODEQL_PATH` - The path to the CodeQL CLI application to use.
> +- `STUART_CODEQL_QUERY_SPECIFIERS` - The CodeQL CLI query specifiers
> to use. See [Running codeql database
> analyze](https://codeql.github.com/docs/codeql-cli/analyzing-databases-
> with-the-codeql-cli/#running-codeql-database-analyze)
> +  for possible options.
> +- `STUART_CODEQL_FILTER_FILES` - The path to "filter" files that contains
> filter patterns as described in
> +  [Filter Patterns](#filter-patterns).
> +  - More than one file may be specified by separating each absolute file path
> with a comma.
> +    - This might be useful to reference a global filter file from an upstream
> repo and also include a global filter
> +      file for the local repo.
> +    - Filters are concatenated in the order of files in the variable. Patterns in
> later files can override patterns
> +      in earlier files.
> +  - The file only needs to contain a list of filter pattern strings under a
> `"Filters"` key. For example:
> +
> +    ```yaml
> +      {
> +        "Filters": [
> +          "<pattern-line-1>",
> +          "<pattern-line-2>"
> +        ]
> +      }
> +      ...
> +    ```
> +
> +    Comments are allowed in the filter files and begin with `#` (like a normal
> YAML file).
> +
> +## Package-Specific Configuration
> +
> +Package-specific configuration values reuse existing package-level
> configuration approaches to simplify adjusting
> +CodeQL plugin behavior per package.
> +
> +These values are all optional. They provide a convenient mechanism for a
> package owner to adjust settings specific to
> +the package.
> +
> +``` yaml
> +  "CodeQlAnalyze": {
> +      "AuditOnly": False,         # Don't fail the build if there are errors. Just log
> them.
> +      "QuerySpecifiers": ""       # Query specifiers to pass to CodeQL CLI.
> +      "Filters": ""               # Inclusion/exclusion filters
> +  }
> +```
> +
> +> _NOTE:_ If a global filter set is provided via
> `STUART_CODEQL_FILTER_FILES` and a package has a package-specific
> +> list, then the package-specific filter list (in a package CI YAML file) is
> appended onto the global filter list and
> +> may be used to override settings in the global list.
> +
> +The format used to specify items in `"Filters"` is specified in [Filter
> Patterns](#filter-patterns).
> +
> +## Filter Patterns
> +
> +As you inspect results, you may want to include or exclude certain sets of
> results. For example, exclude some files by
> +file path entirely or adjust the CodeQL rule applied to a certain file. This
> plugin reuses logic from a popular
> +GitHub Action called [`filter-sarif`](https://github.com/advanced-
> security/filter-sarif) to allow filtering as part of
> +the plugin analysis process.
> +
> +If any results are excluded using filters, the results are removed from the
> SARIF file. This allows the exclude results
> +seen locally to exactly match the results on the CI server.
> +
> +Read the ["Patterns"](https://github.com/advanced-security/filter-
> sarif#patterns) section there for more details. The
> +patterns section is also copied below with some updates to make the
> information more relevant for an edk2 codebase
> +for convenience.
> +
> +Each pattern line is of the form:
> +
> +```plaintext
> +[+/-]<file pattern>[:<rule pattern>]
> +```
> +
> +For example:
> +
> +```yaml
> +-**/*Test*.c:**             # exclusion pattern: remove all alerts from all test files
> +-**/*Test*.c                # ditto, short form of the line above
> ++**/*.c:cpp/infiniteloop    # inclusion pattern: This line has precedence over
> the first two
> +                            # and thus "allow lists" alerts of type "cpp/infiniteloop"
> +**/*.c:cpp/infiniteloop     # ditto, the "+" in inclusion patterns is optional
> +**                          # allow all alerts in all files (reverses all previous lines)
> +```
> +
> +- The path separator character in patterns is always `/`, independent of the
> platform the code is running on and
> +  independent of the paths in the SARIF file.
> +- `*` matches any character, except a path separator
> +- `**` matches any character and is only allowed between path separators,
> e.g. `/**/file.txt`, `**/file.txt` or `**`.
> +  NOT allowed: `**.txt`, `/etc**`
> +- The rule pattern is optional. If omitted, it will apply to alerts of all types.
> +- Subsequent lines override earlier ones. By default all alerts are included.
> +- If you need to use the literals `+`, `-`, `\` or `:` in your pattern, you can
> escape them with `\`, e.g.
> +  `\-this/is/an/inclusion/file/pattern\:with-a-
> semicolon:and/a/rule/pattern/with/a/\\/backslash`. For `+` and `-`, this
> +  is only necessary if they appear at the beginning of the pattern line.
> +
> +## Integration Instructions
> +
> +First, note that most CodeQL CLI operations will take a long time the first
> time they are run. This is due to:
> +
> +1. Downloads - Downloading the CodeQL CLI binary (during `stuart_update`)
> and downloading CodeQL queries during
> +   CodeQL plugin execution
> +2. Cache not established - CodeQL CLI caches data as it performs analysis.
> The first time analysis is performed will
> +   take more time than in the future.
> +
> +Second, these are build plugins. This means a build needs to take place for
> the plugins to run. This typically happens
> +in the following two scenarios:
> +
> +1. `stuart_build` - A single package is built and the build process is started by
> the stuart tools.
> +2. `stuart_ci_build` - A number of packages may be built and the build
> process is started by the `CompilerPlugin`.
> +
> +In any case, each time a package is built, the CodeQL plugins will be run if
> their scopes are active.
> +
> +### Integration Step 1 - Choose Scopes
> +
> +Decide which scopes need to be enabled in your platform, see [Scopes
> Available](#scopes-available).
> +
> +Consider using a build profile to enable CodeQL so developers and pipelines
> can use the profile when they are
> +interested in CodeQL results but in other cases they can easily work without
> CodeQL in the way.
> +
> +Furthermore, build-script specific command-line parameters might be useful
> to control CodeQL scopes and other
> +behavior.
> +
> +#### Scopes Available
> +
> +This CodeQL plugin leverages scopes to control major pieces of functionality.
> Any combination of scopes can be
> +returned from the `GetActiveScopes()` function in the platform settings
> manager to add and remove functionality.
> +
> +Plugin scopes:
> +
> +- `codeql-analyze` - Activate `CodeQlAnalyzePlugin` to perform post-build
> analysis of the last generated database for
> +  the package and target specified.
> +- `codeql-build` - Activate `CodeQlBuildPlugin` to hook the firmware build in
> pre-build such that the build will
> +  generate a CodeQL database during build.
> +
> +In most cases, to perform a full CodeQL run, `codeql-build` should be
> enabled so a new CodeQL database is generated
> +during build and `codeql-analyze` should be be enabled so analysis of that
> database is performed after the build is
> +completed.
> +
> +External dependency scopes:
> +
> +- `codeql-ext-dep` - Downloads the cross-platform CodeQL CLI as an external
> dependency.
> +- `codeql-linux-ext-dep` - Downloads the Linux CodeQL CLI as an external
> dependency.
> +- `codeql-windows-ext-dep` - Downloads the Windows CodeQL CLI as an
> external dependency.
> +
> +Note, that the CodeQL CLI is large in size. Sizes as of the [v2.11.2
> release](https://github.com/github/codeql-cli-binaries/releases/tag/v2.11.2).
> +
> +| Cross-platform |  Linux | Windows |
> +|:--------------:|:------:|:-------:|
> +|     934 MB     | 415 MB |  290 MB |
> +
> +Therefore, the following is recommended:
> +
> +1. **Ideal** - Create container images for build agents and install the
> CodeQL CLI for the container OS into the
> +   container.
> +2. Leverage host-OS detection (e.g.
> [`GetHostInfo()`](https://github.com/tianocore/edk2-pytool-
> library/blob/42ad6561af73ba34564f1577f64f7dbaf1d0a5a2/edk2toollib/utilit
> y_functions.py#L112))
> +to set the scope for the appropriate operating system. This will download
> the much smaller OS-specific application.
> +
> +> _NOTE:_ You should never have more than one CodeQL external
> dependency scope enabled at a time.
> +
> +### Integration Step 2 - Choose CodeQL Queries
> +
> +Determine which queries need to be run against packages in your repo. In
> most cases, the same set of queries will be
> +run against all packages. It is also possible to customize the queries run at
> the package level.
> +
> +The default set of Project Mu CodeQL queries is specified in the
> `MuCodeQlQueries.qls` file in this plugin.
> +
> +> _NOTE:_ The queries in `MuCodeQlQueries.qls` may change at any time. If
> you do not want these changes to impact
> +> your platform, do not relay on option (3).
> +
> +The plugin decides what queries to run based on the following, in order of
> preference:
> +
> +1. Package CI YAML file query specifier
> +2. Build environment variable query specifier
> +3. Plugin default query set file
> +
> +For details on how to set (1) and (2), see the Package CI Configuration and
> Environment Variable sections respectively.
> +
> +> _NOTE:_ The value specified is directly passed as a `query specifier` to
> CodeQL CLI. Therefore, the arguments
> +> allowed by the `<query-specifiers>` argument of CodeQL CLI are allowed
> here. See
> +> [Running codeql database
> analyze](https://codeql.github.com/docs/codeql-cli/analyzing-databases-
> with-the-codeql-cli/#running-codeql-database-analyze).
> +
> +A likely scenario is that a platform needs to run local/closed source queries
> in addition to the open-source queries.
> +There's various ways to handle that:
> +
> +1. Create a query specifier that includes all the queries needed, both public
> and private and use that query specifier,
> +   either globally or at package-level.
> +
> +   For example, at the global level - `STUART_CODEQL_QUERY_SPECIFIERS` =
> _"Absolute_path_to_AllMyQueries.qls"_
> +
> +2. Specify a query specifier that includes the closed sources queries and
> reuse the public query list provided by
> +   this plugin.
> +
> +   For example, at the global level - `STUART_CODEQL_QUERY_SPECIFIERS` =
> _"Absolute_path_to_MuCodeQlQueries.qls
> +   Absolute_path_to_ClosedSourceQueries.qls"_
> +
> +Refer to the CodeQL documentation noted above on query specifiers to
> devise other options.
> +
> +### Integration Step 3 - Determine Global Configuration Values
> +
> +Review the Environment Variable section to determine which, if any, global
> values need to be set in your build script.
> +
> +### Integration Step 4 - Determine Package-Specific Configuration Values
> +
> +Review the Package CI Configuration section to determine which, if any,
> global values need to be set in your
> +package's CI YAML file.
> +
> +### Integration Step 5 - Testing
> +
> +Verify a `stuart_update` and `stuart_build` (or `stuart_ci_build`) command
> work.
> +
> +### Integration Step 6 - Define Inclusion and Exclusion Filter Patterns
> +
> +After reviewing the test results from Step 5, determine if you need to apply
> any filters as described in
> +[Filter Patterns](#filter-patterns).
> +
> +## High-Level Operation
> +
> +This section summarizes the complete CodeQL plugin flow. This is to help
> developers understand basic theory of
> +operation behind the plugin and can be skipped by anyone not interested in
> those details.
> +
> +### CodeQlBuildPlugin
> +
> +1. Register a pre-build hook
> +2. Determine the package and target being built
> +3. Determine the best CodeQL CLI path to use
> +   - First choice, the `STUART_CODEQL_PATH` environment variable
> +     - Note: This is set by the CodeQL CLI external dependency if that is used
> +   - Second choice, `codeql` as found on the system path
> +4. Determine the directory name for the CodeQL database
> +   - Format: `Build/codeql-db-<package>-<target>-<instance>`
> +5. Clean the build directory of the active platform and target
> +   - CodeQL database generation only works on clean builds
> +6. Ensure the "build" step is not skipped as a build is needed to generate a
> CodeQL database
> +7. Build a CodeQL file that wraps around the edk2 build
> +   - Written to the package build directory
> +     - Example: `Build/MdeModulePkg/VS2022/codeql_build_command.bat`
> +8. Set the variables necessary for stuart to call CodeQL CLI during the build
> phase
> +   - Sets `EDK_BUILD_CMD` and `EDK_BUILD_PARAMS`
> +
> +### CodeQlAnalyzePlugin
> +
> +1. Register a post-build hook
> +2. Determine the package and target being built
> +3. Determine the best CodeQL CLI path to use
> +   - First choice, the `STUART_CODEQL_PATH` environment variable
> +     - Note: This is set by the CodeQL CLI external dependency if that is used
> +   - Second choice, `codeql` as found on the system path
> +4. Determine the directory name for the most recent CodeQL database
> +   - Format: `Build/codeql-db-<package>-<target>-<instance>`
> +5. Determine plugin audit status for the given package and target
> +   - Check if `AuditOnly` is enabled either globally or for the package
> +6. Determine the CodeQL query specifiers to use for the given package and
> target
> +   - First choice, the package CI YAML file value
> +   - Second choice, the `STUART_CODEQL_QUERY_SPECIFIERS`
> +   - Third choice, use `CodeQlQueries.qls` (in the plugin directory)
> +7. Run CodeQL CLI to perform database analysis
> +8. Parse the analysis SARIF file to determine the number of CodeQL failures
> +9. Return the number of failures (or zero if `AuditOnly` is enabled)
> +
> +## Local Development Tips
> +
> +This section contains helpful tips to expedite common scenarios when
> working with CodeQL locally.
> +
> +1. Pre-build, Build, and Post-Build
> +
> +   Generating a database requires the pre-build and build steps. Analyzing a
> database requires the post-build step.
> +
> +   Therefore, if you are making tweaks that don't affect the build, such as
> modifying the CodeQL queries used or level
> +   of severity reported, you can save time by skipping pre-build and post-
> build (e.g. `--skipprebuild` and
> +   `--skipbuild`).
> +
> +2. Scopes
> +
> +   Similar to (1), add/remove `codeql-build` and `codeql-analyze` from the
> active scopes to save time depending on what
> +   you are trying to do.
> +
> +   If you are focusing on coding, remove the code CodeQL scopes if they are
> active. If you are ready to check your
> +   changes against CodeQL, simply add the scopes back. It is recommended
> to use build profiles to do this more
> +   conveniently.
> +
> +   If you already have CodeQL CLI enabled, you can remove the `codeql-ext-
> dep` scope locally. The build will use the
> +   `codeql` command on your path.
> +
> +3. CodeQL Output is in the CI Build Log
> +
> +   To see exactly which queries CodeQL ran or why it might be taking longer
> than expected, look in the CI build log
> +   (i.e. `Build/CI_BUILDLOG.txt`) where the CodeQL CLI application output is
> written.
> +
> +   Search for the text you see in the progress output (e.g. "Analyzing
> _MdeModulePkg_ (_DEBUG_) CodeQL database at")
> +   to jump to the section of the log just before the CodeQL CLI is invoked.
> +
> +4. Use a SARIF Viewer to Read Results
> +
> +The [SARIF Viewer extension for VS
> Code](https://marketplace.visualstudio.com/items?itemName=MS-
> SarifVSCode.sarif-viewer)
> +can open the .sarif file generated by this plugin and allow you to click links
> directly to the problem area in source
> +files.
> +
> +## Resolution Guidelines
> +
> +This section captures brief guidelines to keep in mind while resolving
> CodeQL issues.
> +
> +1. Look at surrounding code. Changes should always take into account the
> context of nearby code. The new logic may
> +   need to account conditions not immediately obvious based on the issue
> alone. It is easy to focus only on the line
> +   of code highlighted by CodeQL and miss the code's role in the big picture.
> +2. A CodeQL alert may be benign but the code can be refactored to prevent
> the alert. Often refactoring the code makes
> +   the code intention clearer and avoids an unnecessary exception.
> +3. Consider adding unit tests while making CodeQL fixes especially for
> commonly used code and code with a high volume
> +   of CodeQL alerts.
> diff --git a/BaseTools/Plugin/CodeQL/analyze/__init__.py
> b/BaseTools/Plugin/CodeQL/analyze/__init__.py
> new file mode 100644
> index 000000000000..e69de29bb2d1
> diff --git a/BaseTools/Plugin/CodeQL/analyze/analyze_filter.py
> b/BaseTools/Plugin/CodeQL/analyze/analyze_filter.py
> new file mode 100644
> index 000000000000..9a544e3192c6
> --- /dev/null
> +++ b/BaseTools/Plugin/CodeQL/analyze/analyze_filter.py
> @@ -0,0 +1,176 @@
> +# @file analyze_filter.py
> +#
> +# Filters results in a SARIF file.
> +#
> +# Based on code in:
> +#   https://github.com/advanced-security/filter-sarif
> +#
> +# Specifically:
> +#   https://github.com/advanced-security/filter-
> sarif/blob/main/filter_sarif.py
> +#
> +# That code is licensed under:
> +#            Apache License
> +#      Version 2.0, January 2004
> +#   http://www.apache.org/licenses/
> +#
> +# View the full and complete license as provided by that repository here:
> +#   https://github.com/advanced-security/filter-sarif/blob/main/LICENSE
> +#
> +# This file has been altered from its original form.
> +#
> +# It primarily contains modifications made to integrate with the CodeQL
> plugin.
> +#
> +# Copyright (c) Microsoft Corporation. All rights reserved.
> +# SPDX-License-Identifier: BSD-2-Clause-Patent
> +##
> +
> +import json
> +import logging
> +import re
> +from os import PathLike
> +from typing import Iterable, List, Tuple
> +
> +from analyze.globber import match
> +
> +
> +def _match_path_and_rule(
> +    path: str, rule: str, patterns: Iterable[str]) -> bool:
> +    """Returns whether a given path matches a given rule.
> +
> +    Args:
> +        path (str): A file path string.
> +        rule (str): A rule file path string.
> +        patterns (Iterable[str]): An iterable of pattern strings.
> +
> +    Returns:
> +        bool: True if the path matches a rule. Otherwise, False.
> +    """
> +    result = True
> +    for s, fp, rp in patterns:
> +        if match(rp, rule) and match(fp, path):
> +            result = s
> +    return result
> +
> +
> +def _parse_pattern(line: str) -> Tuple[str]:
> +    """Parses a given pattern line.
> +
> +    Args:
> +        line (str): The line string that contains the rule.
> +
> +    Returns:
> +        Tuple[str]: The parsed sign, file pattern, and rule pattern from the
> +                    line.
> +    """
> +    sep_char = ':'
> +    esc_char = '\\'
> +    file_pattern = ''
> +    rule_pattern = ''
> +    seen_separator = False
> +    sign = True
> +
> +    # inclusion or exclusion pattern?
> +    u_line = line
> +    if line:
> +        if line[0] == '-':
> +            sign = False
> +            u_line = line[1:]
> +        elif line[0] == '+':
> +            u_line = line[1:]
> +
> +    i = 0
> +    while i < len(u_line):
> +        c = u_line[i]
> +        i = i + 1
> +        if c == sep_char:
> +            if seen_separator:
> +                raise Exception(
> +                    'Invalid pattern: "' + line + '" Contains more than one '
> +                    'separator!')
> +            seen_separator = True
> +            continue
> +        elif c == esc_char:
> +            next_c = u_line[i] if (i < len(u_line)) else None
> +            if next_c in ['+' , '-', esc_char, sep_char]:
> +                i = i + 1
> +                c = next_c
> +        if seen_separator:
> +            rule_pattern = rule_pattern + c
> +        else:
> +            file_pattern = file_pattern + c
> +
> +    if not rule_pattern:
> +        rule_pattern = '**'
> +
> +    return sign, file_pattern, rule_pattern
> +
> +
> +def filter_sarif(input_sarif: PathLike,
> +                 output_sarif: PathLike,
> +                 patterns: List[str],
> +                 split_lines: bool) -> None:
> +    """Filters a SARIF file with a given set of filter patterns.
> +
> +    Args:
> +        input_sarif (PathLike): Input SARIF file path.
> +        output_sarif (PathLike): Output SARIF file path.
> +        patterns (PathLike): List of filter pattern strings.
> +        split_lines (PathLike): Whether to split lines in individual patterns.
> +    """
> +    if split_lines:
> +        tmp = []
> +        for p in patterns:
> +            tmp = tmp + re.split('\r?\n', p)
> +        patterns = tmp
> +
> +    patterns = [_parse_pattern(p) for p in patterns if p]
> +
> +    logging.debug('Given patterns:')
> +    for s, fp, rp in patterns:
> +        logging.debug(
> +            'files: {file_pattern}    rules: {rule_pattern} ({sign})'.format(
> +                file_pattern=fp,
> +                rule_pattern=rp,
> +                sign='positive' if s else 'negative'))
> +
> +    with open(input_sarif, 'r') as f:
> +        s = json.load(f)
> +
> +    for run in s.get('runs', []):
> +        if run.get('results', []):
> +            new_results = []
> +            for r in run['results']:
> +                if r.get('locations', []):
> +                    new_locations = []
> +                    for l in r['locations']:
> +                        # TODO: The uri field is optional. We might have to
> +                        #       fetch the actual uri from "artifacts" via
> +                        #       "index"
> +                        # (see https://github.com/microsoft/sarif-
> tutorials/blob/main/docs/2-Basics.md#-linking-results-to-artifacts)
> +                        uri = l.get(
> +                                    'physicalLocation', {}).get(
> +                                        'artifactLocation', {}).get(
> +                                            'uri', None)
> +
> +                        # TODO: The ruleId field is optional and potentially
> +                        #       ambiguous. We might have to fetch the actual
> +                        #       ruleId from the rule metadata via the ruleIndex
> +                        #       field.
> +                        # (see https://github.com/microsoft/sarif-
> tutorials/blob/main/docs/2-Basics.md#rule-metadata)
> +                        ruleId = r['ruleId']
> +
> +                        if (uri is None or
> +                            _match_path_and_rule(uri, ruleId, patterns)):
> +                            new_locations.append(l)
> +                    r['locations'] = new_locations
> +                    if new_locations:
> +                        new_results.append(r)
> +                else:
> +                    # locations array doesn't exist or is empty, so we can't
> +                    # match on anything. Therefore, we include the result in
> +                    # the output.
> +                    new_results.append(r)
> +            run['results'] = new_results
> +
> +    with open(output_sarif, 'w') as f:
> +        json.dump(s, f, indent=2)
> diff --git a/BaseTools/Plugin/CodeQL/analyze/globber.py
> b/BaseTools/Plugin/CodeQL/analyze/globber.py
> new file mode 100644
> index 000000000000..25548fc9c754
> --- /dev/null
> +++ b/BaseTools/Plugin/CodeQL/analyze/globber.py
> @@ -0,0 +1,132 @@
> +# @file globber.py
> +#
> +# Provides global functionality for use by the CodeQL plugin.
> +#
> +# Copyright 2019 Jaakko Kangasharju
> +#
> +# Licensed under the Apache License, Version 2.0 (the "License");
> +# you may not use this file except in compliance with the License.
> +# You may obtain a copy of the License at
> +#
> +#    http://www.apache.org/licenses/LICENSE-2.0
> +#
> +# Unless required by applicable law or agreed to in writing, software
> +# distributed under the License is distributed on an "AS IS" BASIS,
> +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> +# See the License for the specific language governing permissions and
> +# limitations under the License.
> +#
> +# This file has been altered from its original form.
> +#
> +# Based on code in:
> +#   https://github.com/advanced-security/filter-sarif
> +#
> +# Specifically:
> +#   https://github.com/advanced-security/filter-
> sarif/blob/main/filter_sarif.py
> +#
> +# That code is licensed under:
> +#            Apache License
> +#      Version 2.0, January 2004
> +#   http://www.apache.org/licenses/
> +#
> +# This file has been altered from its original form. Primarily modifications
> +# made to integrate with the CodeQL plugin.
> +#
> +# Copyright (c) Microsoft Corporation. All rights reserved.
> +# SPDX-License-Identifier: BSD-2-Clause-Patent
> +##
> +
> +import re
> +
> +_double_star_after_invalid_regex = re.compile(r'[^/\\]\*\*')
> +_double_star_first_before_invalid_regex = re.compile('^\\*\\*[^/]')
> +_double_star_middle_before_invalid_regex = re.compile(r'[^\\]\*\*[^/]')
> +
> +
> +def _match_component(pattern_component, file_name_component):
> +    if len(pattern_component) == 0 and len(file_name_component) == 0:
> +        return True
> +    elif len(pattern_component) == 0:
> +        return False
> +    elif len(file_name_component) == 0:
> +        return pattern_component == '*'
> +    elif pattern_component[0] == '*':
> +        return (_match_component(pattern_component,
> file_name_component[1:]) or
> +                _match_component(pattern_component[1:],
> file_name_component))
> +    elif pattern_component[0] == '?':
> +        return _match_component(pattern_component[1:],
> file_name_component[1:])
> +    elif pattern_component[0] == '\\':
> +        return (len(pattern_component) >= 2 and
> +                pattern_component[1] == file_name_component[0] and
> +                _match_component(
> +                    pattern_component[2:], file_name_component[1:]))
> +    elif pattern_component[0] != file_name_component[0]:
> +        return False
> +    else:
> +        return _match_component(pattern_component[1:],
> file_name_component[1:])
> +
> +
> +def _match_components(pattern_components, file_name_components):
> +    if len(pattern_components) == 0 and len(file_name_components) == 0:
> +        return True
> +    if len(pattern_components) == 0:
> +        return False
> +    if len(file_name_components) == 0:
> +        return len(pattern_components) == 1 and pattern_components[0] ==
> '**'
> +    if pattern_components[0] == '**':
> +        return (_match_components(pattern_components,
> file_name_components[1:])
> +                or _match_components(
> +                    pattern_components[1:], file_name_components))
> +    else:
> +        return (
> +            _match_component(
> +                pattern_components[0], file_name_components[0]) and
> +            _match_components(
> +                pattern_components[1:], file_name_components[1:]))
> +
> +
> +def match(pattern: str, file_name: str):
> +    """Match a glob pattern against a file name.
> +
> +    Glob pattern matching is for file names, which do not need to exist as
> +    files on the file system.
> +
> +    A file name is a sequence of directory names, possibly followed by the
> name
> +    of a file, with the components separated by a path separator. A glob
> +    pattern is similar, except it may contain special characters: A '?' matches
> +    any character in a name. A '*' matches any sequence of characters
> (possibly
> +    empty) in a name. Both of these match only within a single component,
> i.e.,
> +    they will not match a path separator. A component in a pattern may also
> be
> +    a literal '**', which matches zero or more components in the complete file
> +    name. A backslash '\\' in a pattern acts as an escape character, and
> +    indicates that the following character is to be matched literally, even if
> +    it is a special character.
> +
> +    Args:
> +        pattern (str): The pattern to match. The path separator in patterns is
> +                       always '/'.
> +        file_name (str): The file name to match against. The path separator in
> +                         file names is the platform separator
> +
> +    Returns:
> +        bool: True if the pattern matches, False otherwise.
> +    """
> +    if (_double_star_after_invalid_regex.search(pattern) is not None or
> +        _double_star_first_before_invalid_regex.search(
> +            pattern) is not None or
> +        _double_star_middle_before_invalid_regex.search(pattern) is not
> None):
> +        raise ValueError(
> +            '** in {} not alone between path separators'.format(pattern))
> +
> +    pattern = pattern.rstrip('/')
> +    file_name = file_name.rstrip('/')
> +
> +    while '**/**' in pattern:
> +        pattern = pattern.replace('**/**', '**')
> +
> +    pattern_components = pattern.split('/')
> +
> +    # We split on '\' as well as '/' to support unix and windows-style paths
> +    file_name_components = re.split(r'[\\/]', file_name)
> +
> +    return _match_components(pattern_components,
> file_name_components)
> diff --git a/BaseTools/Plugin/CodeQL/codeqlcli_ext_dep.yaml
> b/BaseTools/Plugin/CodeQL/codeqlcli_ext_dep.yaml
> new file mode 100644
> index 000000000000..37c7c9f595ca
> --- /dev/null
> +++ b/BaseTools/Plugin/CodeQL/codeqlcli_ext_dep.yaml
> @@ -0,0 +1,26 @@
> +## @file codeqlcli_ext_dep.yaml
> +#
> +# Downloads the CodeQL Command-Line Interface (CLI) application that
> support Linux, Windows, and Mac OS X.
> +#
> +# This download is very large but conveniently provides support for all
> operating systems. Use it if you
> +# need CodeQL CLI support without concern for the host operating system.
> +#
> +# In an environment where a platform might build in different operating
> systems, it is recommended to set
> +# the scope for the appropriate CodeQL external dependency based on the
> host operating system being used.
> +#
> +# Copyright (c) Microsoft Corporation. All rights reserved.
> +# SPDX-License-Identifier: BSD-2-Clause-Patent
> +##
> +
> +{
> +  "scope": "codeql-ext-dep",
> +  "type": "web",
> +  "name": "codeql_cli",
> +  "source": "https://github.com/github/codeql-cli-
> binaries/releases/download/v2.12.4/codeql.zip",
> +  "version": "2.12.4",
> +  "sha256":
> "f682f1155d627ad97f10b1bcad97f682011986717bd3823e9cf831ed83ac96e7"
> ,
> +  "compression_type": "zip",
> +  "internal_path": "/codeql/",
> +  "flags": ["set_shell_var", ],
> +  "var_name": "STUART_CODEQL_PATH"
> +}
> diff --git a/BaseTools/Plugin/CodeQL/codeqlcli_linux_ext_dep.yaml
> b/BaseTools/Plugin/CodeQL/codeqlcli_linux_ext_dep.yaml
> new file mode 100644
> index 000000000000..a6ca5d0f34cc
> --- /dev/null
> +++ b/BaseTools/Plugin/CodeQL/codeqlcli_linux_ext_dep.yaml
> @@ -0,0 +1,24 @@
> +## @file codeqlcli_linux_ext_dep.yaml
> +#
> +# Downloads the Linux CodeQL Command-Line Interface (CLI) application.
> +#
> +# This download only supports Linux. In an environment where a platform
> might build in different operating
> +# systems, it is recommended to set the scope for the appropriate CodeQL
> external dependency based on the
> +# host operating system being used.
> +#
> +# Copyright (c) Microsoft Corporation. All rights reserved.
> +# SPDX-License-Identifier: BSD-2-Clause-Patent
> +##
> +
> +{
> +  "scope": "codeql-linux-ext-dep",
> +  "type": "web",
> +  "name": "codeql_linux_cli",
> +  "source": "https://github.com/github/codeql-cli-
> binaries/releases/download/v2.14.5/codeql-linux64.zip",
> +  "version": "2.14.5",
> +  "sha256":
> "72aa5d748ff9ab57cfd86045560683bdc4897e0fe6d9f9a2786d9394674ae733"
> ,
> +  "compression_type": "zip",
> +  "internal_path": "/codeql/",
> +  "flags": ["set_shell_var", ],
> +  "var_name": "STUART_CODEQL_PATH"
> +}
> diff --git a/BaseTools/Plugin/CodeQL/codeqlcli_windows_ext_dep.yaml
> b/BaseTools/Plugin/CodeQL/codeqlcli_windows_ext_dep.yaml
> new file mode 100644
> index 000000000000..e706a7cabf9f
> --- /dev/null
> +++ b/BaseTools/Plugin/CodeQL/codeqlcli_windows_ext_dep.yaml
> @@ -0,0 +1,24 @@
> +## @file codeqlcli_windows_ext_dep.yaml
> +#
> +# Downloads the Windows CodeQL Command-Line Interface (CLI)
> application.
> +#
> +# This download only supports Windows. In an environment where a
> platform might build in different operating
> +# systems, it is recommended to set the scope for the appropriate CodeQL
> external dependency based on the
> +# host operating system being used.
> +#
> +# Copyright (c) Microsoft Corporation. All rights reserved.
> +# SPDX-License-Identifier: BSD-2-Clause-Patent
> +##
> +
> +{
> +  "scope": "codeql-windows-ext-dep",
> +  "type": "web",
> +  "name": "codeql_windows_cli",
> +  "source": "https://github.com/github/codeql-cli-
> binaries/releases/download/v2.14.5/codeql-win64.zip",
> +  "version": "2.14.5",
> +  "sha256":
> "861fcb38365cc311efee0c3a28c77494e93c69a969885b72e53173ad473f61aa",
> +  "compression_type": "zip",
> +  "internal_path": "/codeql/",
> +  "flags": ["set_shell_var", ],
> +  "var_name": "STUART_CODEQL_PATH"
> +}
> diff --git a/BaseTools/Plugin/CodeQL/common/__init__.py
> b/BaseTools/Plugin/CodeQL/common/__init__.py
> new file mode 100644
> index 000000000000..e69de29bb2d1
> diff --git a/BaseTools/Plugin/CodeQL/common/codeql_plugin.py
> b/BaseTools/Plugin/CodeQL/common/codeql_plugin.py
> new file mode 100644
> index 000000000000..c827cc30ae8b
> --- /dev/null
> +++ b/BaseTools/Plugin/CodeQL/common/codeql_plugin.py
> @@ -0,0 +1,74 @@
> +# @file codeql_plugin.py
> +#
> +# Common logic shared across the CodeQL plugin.
> +#
> +# Copyright (c) Microsoft Corporation. All rights reserved.
> +# SPDX-License-Identifier: BSD-2-Clause-Patent
> +##
> +
> +import os
> +import shutil
> +from os import PathLike
> +
> +from edk2toollib.utility_functions import GetHostInfo
> +
> +
> +def get_codeql_db_path(workspace: PathLike, package: str, target: str,
> +                       new_path: bool = True) -> str:
> +    """Return the CodeQL database path for this build.
> +
> +    Args:
> +        workspace (PathLike): The workspace path.
> +        package (str): The package name (e.g. "MdeModulePkg")
> +        target (str): The target (e.g. "DEBUG")
> +        new_path (bool, optional): Whether to create a new database path or
> +                                   return an existing path. Defaults to True.
> +
> +    Returns:
> +        str: The absolute path to the CodeQL database directory.
> +    """
> +    codeql_db_dir_name = "codeql-db-" + package + "-" + target
> +    codeql_db_dir_name = codeql_db_dir_name.lower()
> +    codeql_db_path = os.path.join("Build", codeql_db_dir_name)
> +    codeql_db_path = os.path.join(workspace, codeql_db_path)
> +
> +    i = 0
> +    while os.path.isdir(f"{codeql_db_path + '-%s' % i}"):
> +        i += 1
> +
> +    if not new_path:
> +        if i == 0:
> +            return None
> +        else:
> +            i -= 1
> +
> +    return codeql_db_path + f"-{i}"
> +
> +
> +def get_codeql_cli_path() -> str:
> +    """Return the current CodeQL CLI path.
> +
> +    Returns:
> +        str: The absolute path to the CodeQL CLI application to use for
> +             this build.
> +    """
> +    # The CodeQL executable path can be passed via the
> +    # STUART_CODEQL_PATH environment variable (to override with a
> +    # custom value for this run) or read from the system path.
> +    codeql_path = None
> +
> +    if "STUART_CODEQL_PATH" in os.environ:
> +        codeql_path = os.environ["STUART_CODEQL_PATH"]
> +
> +        if GetHostInfo().os == "Windows":
> +            codeql_path = os.path.join(codeql_path, "codeql.exe")
> +        else:
> +            codeql_path = os.path.join(codeql_path, "codeql")
> +
> +        if not os.path.isfile(codeql_path):
> +            codeql_path = None
> +
> +    if not codeql_path:
> +        codeql_path = shutil.which("codeql")
> +
> +    return codeql_path
> --
> 2.42.0.windows.2



-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.
View/Reply Online (#109996): https://edk2.groups.io/g/devel/message/109996
Mute This Topic: https://groups.io/mt/102031057/7686176
Group Owner: devel+owner@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub [rebecca@openfw.io]
-=-=-=-=-=-=-=-=-=-=-=-