From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mga01.intel.com (mga01.intel.com []) by mx.groups.io with SMTP id smtpd.web11.1593.1573461678580668089 for ; Mon, 11 Nov 2019 00:41:19 -0800 Authentication-Results: mx.groups.io; dkim=missing; spf=fail (domain: intel.com, ip: , mailfrom: michael.d.kinney@intel.com) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga101.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 11 Nov 2019 00:41:13 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.68,292,1569308400"; d="scan'208";a="234403648" Received: from mdkinney-mobl2.amr.corp.intel.com ([10.255.231.122]) by fmsmga002.fm.intel.com with ESMTP; 11 Nov 2019 00:41:12 -0800 From: "Michael D Kinney" To: devel@edk2.groups.io Cc: Sean Brogan , Bob Feng , Liming Gao Subject: [Patch v5 06/22] BaseTools: Add BaseTools plugins to support CI Date: Mon, 11 Nov 2019 00:40:51 -0800 Message-Id: <20191111084107.5368-7-michael.d.kinney@intel.com> X-Mailer: git-send-email 2.21.0.windows.1 In-Reply-To: <20191111084107.5368-1-michael.d.kinney@intel.com> References: <20191111084107.5368-1-michael.d.kinney@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit From: Sean Brogan Add the following plugins that are required to support EDK II Continuous Integration (CI) builds. These plugins are added to BaseTools because that support EDK II BaseTools features. * BuildToolsReportGenerator * LinuxGcc5ToolChain * WindowsResourceCompiler * WindowsVsToolChain Cc: Bob Feng Cc: Liming Gao Signed-off-by: Michael D Kinney Reviewed-by: Liming Gao --- .../BuildToolsReportGenerator.py | 69 ++++++++++ .../BuildToolsReportGenerator_plug_in.yaml | 12 ++ .../BuildToolsReport_Template.html | 126 ++++++++++++++++++ .../LinuxGcc5ToolChain/LinuxGcc5ToolChain.py | 85 ++++++++++++ .../LinuxGcc5ToolChain_plug_in.yaml | 12 ++ .../WindowsResourceCompiler/WinRcPath.py | 29 ++++ .../WinRcPath_plug_in.yaml | 13 ++ .../WindowsVsToolChain/WindowsVsToolChain.py | 126 ++++++++++++++++++ .../WindowsVsToolChain_plug_in.yaml | 11 ++ 9 files changed, 483 insertions(+) create mode 100644 BaseTools/Plugin/BuildToolsReport/BuildToolsReportGenerator.py create mode 100644 BaseTools/Plugin/BuildToolsReport/BuildToolsReportGenerator_plug_in.yaml create mode 100644 BaseTools/Plugin/BuildToolsReport/BuildToolsReport_Template.html create mode 100644 BaseTools/Plugin/LinuxGcc5ToolChain/LinuxGcc5ToolChain.py create mode 100644 BaseTools/Plugin/LinuxGcc5ToolChain/LinuxGcc5ToolChain_plug_in.yaml create mode 100644 BaseTools/Plugin/WindowsResourceCompiler/WinRcPath.py create mode 100644 BaseTools/Plugin/WindowsResourceCompiler/WinRcPath_plug_in.yaml create mode 100644 BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain.py create mode 100644 BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain_plug_in.yaml diff --git a/BaseTools/Plugin/BuildToolsReport/BuildToolsReportGenerator.py b/BaseTools/Plugin/BuildToolsReport/BuildToolsReportGenerator.py new file mode 100644 index 0000000000..d5df9f2f9b --- /dev/null +++ b/BaseTools/Plugin/BuildToolsReport/BuildToolsReportGenerator.py @@ -0,0 +1,69 @@ +## +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import os +import logging +import json + +try: + from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin + + class BuildToolsReportGenerator(IUefiBuildPlugin): + def do_report(self, thebuilder): + try: + from edk2toolext.environment import version_aggregator + except ImportError: + logging.critical("Loading BuildToolsReportGenerator failed, please update your Edk2-PyTool-Extensions") + return 0 + + OutputReport = os.path.join(thebuilder.env.GetValue("BUILD_OUTPUT_BASE"), "BUILD_TOOLS_REPORT") + OutputReport = os.path.normpath(OutputReport) + if not os.path.isdir(os.path.dirname(OutputReport)): + os.makedirs(os.path.dirname(OutputReport)) + + Report = BuildToolsReport() + Report.MakeReport(version_aggregator.GetVersionAggregator().GetAggregatedVersionInformation(), OutputReport=OutputReport) + + def do_pre_build(self, thebuilder): + self.do_report(thebuilder) + return 0 + + def do_post_build(self, thebuilder): + self.do_report(thebuilder) + return 0 + +except ImportError: + pass + + +class BuildToolsReport(object): + MY_FOLDER = os.path.dirname(os.path.realpath(__file__)) + VERSION = "1.00" + + def __init__(self): + pass + + def MakeReport(self, BuildTools, OutputReport="BuildToolsReport"): + logging.info("Writing BuildToolsReports to {0}".format(OutputReport)) + versions_list = [] + for key, value in BuildTools.items(): + versions_list.append(value) + versions_list = sorted(versions_list, key=lambda k: k['type']) + json_dict = {"modules": versions_list, + "PluginVersion": BuildToolsReport.VERSION} + + htmlfile = open(OutputReport + ".html", "w") + jsonfile = open(OutputReport + ".json", "w") + template = open(os.path.join(BuildToolsReport.MY_FOLDER, "BuildToolsReport_Template.html"), "r") + + for line in template.readlines(): + if "%TO_BE_FILLED_IN_BY_PYTHON_SCRIPT%" in line: + line = line.replace("%TO_BE_FILLED_IN_BY_PYTHON_SCRIPT%", json.dumps(json_dict)) + htmlfile.write(line) + + jsonfile.write(json.dumps(versions_list, indent=4)) + + jsonfile.close() + template.close() + htmlfile.close() diff --git a/BaseTools/Plugin/BuildToolsReport/BuildToolsReportGenerator_plug_in.yaml b/BaseTools/Plugin/BuildToolsReport/BuildToolsReportGenerator_plug_in.yaml new file mode 100644 index 0000000000..9e07c4889b --- /dev/null +++ b/BaseTools/Plugin/BuildToolsReport/BuildToolsReportGenerator_plug_in.yaml @@ -0,0 +1,12 @@ +## @file +# Build Plugin used to output html report of all versions collected +# during the build +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "global", + "name": "Build Tools Report Generator", + "module": "BuildToolsReportGenerator" +} diff --git a/BaseTools/Plugin/BuildToolsReport/BuildToolsReport_Template.html b/BaseTools/Plugin/BuildToolsReport/BuildToolsReport_Template.html new file mode 100644 index 0000000000..26350ab425 --- /dev/null +++ b/BaseTools/Plugin/BuildToolsReport/BuildToolsReport_Template.html @@ -0,0 +1,126 @@ + + + + + + Build Tools Report + + + + + + +

Build Tools Report

Tools
About

+ + + + + + + + + +

Key	Value	Type

+ Build Tools Report Template Version: 1.00
+ Build Tools Report Plugin Version:
+

License

External Licenses

+ + + + + + + + + + + + diff --git a/BaseTools/Plugin/LinuxGcc5ToolChain/LinuxGcc5ToolChain.py b/BaseTools/Plugin/LinuxGcc5ToolChain/LinuxGcc5ToolChain.py new file mode 100644 index 0000000000..c31641e931 --- /dev/null +++ b/BaseTools/Plugin/LinuxGcc5ToolChain/LinuxGcc5ToolChain.py @@ -0,0 +1,85 @@ +# @file LinuxGcc5ToolChain.py +# Plugin to configures paths for GCC5 ARM/AARCH64 Toolchain +## +# This plugin works in conjuncture with the tools_def +# +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import os +import logging +from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin +from edk2toolext.environment import shell_environment + + +class LinuxGcc5ToolChain(IUefiBuildPlugin): + + def do_post_build(self, thebuilder): + return 0 + + def do_pre_build(self, thebuilder): + self.Logger = logging.getLogger("LinuxGcc5ToolChain") + + # + # GCC5 - The ARM and AARCH64 compilers need their paths set if available + if thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "GCC5": + + # Start with AARACH64 compiler + ret = self._check_aarch64() + if ret != 0: + self.Logger.critical("Failed in check aarch64") + return ret + + # Check arm compiler + ret = self._check_arm() + if ret != 0: + self.Logger.critical("Failed in check arm") + return ret + + return 0 + + def _check_arm(self): + # check to see if full path already configured + if shell_environment.GetEnvironment().get_shell_var("GCC5_ARM_PREFIX") is not None: + self.Logger.info("GCC5_ARM_PREFIX is already set.") + + else: + # now check for install dir. If set then set the Prefix + install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_ARM_INSTALL") + if install_path is None: + return 0 + + # make GCC5_ARM_PREFIX to align with tools_def.txt + prefix = os.path.join(install_path, "bin", "arm-linux-gnueabihf-") + shell_environment.GetEnvironment().set_shell_var("GCC5_ARM_PREFIX", prefix) + + # now confirm it exists + if not os.path.exists(shell_environment.GetEnvironment().get_shell_var("GCC5_ARM_PREFIX") + "gcc"): + self.Logger.error("Path for GCC5_ARM_PREFIX toolchain is invalid") + return -2 + + return 0 + + def _check_aarch64(self): + # check to see if full path already configured + if shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_PREFIX") is not None: + self.Logger.info("GCC5_AARCH64_PREFIX is already set.") + + else: + # now check for install dir. If set then set the Prefix + install_path = shell_environment.GetEnvironment( + ).get_shell_var("GCC5_AARCH64_INSTALL") + if install_path is None: + return 0 + + # make GCC5_AARCH64_PREFIX to align with tools_def.txt + prefix = os.path.join(install_path, "bin", "aarch64-linux-gnu-") + shell_environment.GetEnvironment().set_shell_var("GCC5_AARCH64_PREFIX", prefix) + + # now confirm it exists + if not os.path.exists(shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_PREFIX") + "gcc"): + self.Logger.error( + "Path for GCC5_AARCH64_PREFIX toolchain is invalid") + return -2 + + return 0 diff --git a/BaseTools/Plugin/LinuxGcc5ToolChain/LinuxGcc5ToolChain_plug_in.yaml b/BaseTools/Plugin/LinuxGcc5ToolChain/LinuxGcc5ToolChain_plug_in.yaml new file mode 100644 index 0000000000..aee971606b --- /dev/null +++ b/BaseTools/Plugin/LinuxGcc5ToolChain/LinuxGcc5ToolChain_plug_in.yaml @@ -0,0 +1,12 @@ +## @file +# Build Plugin used to set the path +# for the GCC5 ARM/AARCH64 downloaded compilers +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "global-nix", + "name": "Linux GCC5 Tool Chain Support", + "module": "LinuxGcc5ToolChain" +} diff --git a/BaseTools/Plugin/WindowsResourceCompiler/WinRcPath.py b/BaseTools/Plugin/WindowsResourceCompiler/WinRcPath.py new file mode 100644 index 0000000000..ec2f2d1298 --- /dev/null +++ b/BaseTools/Plugin/WindowsResourceCompiler/WinRcPath.py @@ -0,0 +1,29 @@ +## @file WinRcPath.py +# Plugin to find Windows SDK Resource Compiler rc.exe +## +# This plugin works in conjuncture with the tools_def to support rc.exe +# +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import os +from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin +import edk2toollib.windows.locate_tools as locate_tools +from edk2toolext.environment import shell_environment +from edk2toolext.environment import version_aggregator + +class WinRcPath(IUefiBuildPlugin): + + def do_post_build(self, thebuilder): + return 0 + + def do_pre_build(self, thebuilder): + #get the locate tools module + path = locate_tools.FindToolInWinSdk("rc.exe") + if path is None: + thebuilder.logging.warning("Failed to find rc.exe") + else: + p = os.path.abspath(os.path.dirname(path)) + shell_environment.GetEnvironment().set_shell_var("WINSDK_PATH_FOR_RC_EXE", p) + version_aggregator.GetVersionAggregator().ReportVersion("WINSDK_PATH_FOR_RC_EXE", p, version_aggregator.VersionTypes.INFO) + return 0 diff --git a/BaseTools/Plugin/WindowsResourceCompiler/WinRcPath_plug_in.yaml b/BaseTools/Plugin/WindowsResourceCompiler/WinRcPath_plug_in.yaml new file mode 100644 index 0000000000..253f22f833 --- /dev/null +++ b/BaseTools/Plugin/WindowsResourceCompiler/WinRcPath_plug_in.yaml @@ -0,0 +1,13 @@ +## @file +# Build Plugin used to set the path to rc.exe on windows. +# The plugin is able to use python to locate the tool as to avoid +# hard-coding the path +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "global-win", + "name": "Windows RC Path Support", + "module": "WinRcPath" +} diff --git a/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain.py b/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain.py new file mode 100644 index 0000000000..a8202e5992 --- /dev/null +++ b/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain.py @@ -0,0 +1,126 @@ +## @file WindowsVsToolChain.py +# Plugin to configures paths for the VS2017 and VS2019 tool chain +## +# This plugin works in conjuncture with the tools_def +# +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import os +import logging +from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin +import edk2toollib.windows.locate_tools as locate_tools +from edk2toollib.windows.locate_tools import FindWithVsWhere +from edk2toolext.environment import shell_environment +from edk2toolext.environment import version_aggregator + +class WindowsVsToolChain(IUefiBuildPlugin): + + def do_post_build(self, thebuilder): + return 0 + + def do_pre_build(self, thebuilder): + self.Logger = logging.getLogger("WindowsVsToolChain") + +# + # VS2017 - Follow VS2017 where there is potential for many versions of the tools. + # If a specific version is required then the user must set both env variables: + ## VS150INSTALLPATH: base install path on system to VC install dir. Here you will find the VC folder, etc + ## VS150TOOLVER: version number for the VC compiler tools + ## VS2017_PREFIX: path to MSVC compiler folder with trailing slash (can be used instead of two vars above) + if thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "VS2017": + + # check to see if full path already configured + if shell_environment.GetEnvironment().get_shell_var("VS2017_PREFIX") != None: + self.Logger.info("VS2017_PREFIX is already set.") + + else: + install_path = self._get_vs_install_path("VS2017".lower(), "VS150INSTALLPATH") + vc_ver = self._get_vc_version(install_path, "VS150TOOLVER") + + if install_path is None or vc_ver is None: + self.Logger.error("Failed to configure environment for VS2017") + return -1 + + version_aggregator.GetVersionAggregator().ReportVersion( + "Visual Studio Install Path", install_path, version_aggregator.VersionTypes.INFO) + version_aggregator.GetVersionAggregator().ReportVersion( + "VC Version", vc_ver, version_aggregator.VersionTypes.TOOL) + + #make VS2017_PREFIX to align with tools_def.txt + prefix = os.path.join(install_path, "VC", "Tools", "MSVC", vc_ver) + prefix = prefix + os.path.sep + shell_environment.GetEnvironment().set_shell_var("VS2017_PREFIX", prefix) + + # now confirm it exists + if not os.path.exists(shell_environment.GetEnvironment().get_shell_var("VS2017_PREFIX")): + self.Logger.error("Path for VS2017 toolchain is invalid") + return -2 + + # + # VS2019 - Follow VS2019 where there is potential for many versions of the tools. + # If a specific version is required then the user must set both env variables: + ## VS160INSTALLPATH: base install path on system to VC install dir. Here you will find the VC folder, etc + ## VS160TOOLVER: version number for the VC compiler tools + ## VS2019_PREFIX: path to MSVC compiler folder with trailing slash (can be used instead of two vars above) + elif thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "VS2019": + + # check to see if full path already configured + if shell_environment.GetEnvironment().get_shell_var("VS2019_PREFIX") != None: + self.Logger.info("VS2019_PREFIX is already set.") + + else: + install_path = self._get_vs_install_path("VS2019".lower(), "VS160INSTALLPATH") + vc_ver = self._get_vc_version(install_path, "VS160TOOLVER") + + if install_path is None or vc_ver is None: + self.Logger.error("Failed to configure environment for VS2019") + return -1 + + version_aggregator.GetVersionAggregator().ReportVersion( + "Visual Studio Install Path", install_path, version_aggregator.VersionTypes.INFO) + version_aggregator.GetVersionAggregator().ReportVersion( + "VC Version", vc_ver, version_aggregator.VersionTypes.TOOL) + + #make VS2019_PREFIX to align with tools_def.txt + prefix = os.path.join(install_path, "VC", "Tools", "MSVC", vc_ver) + prefix = prefix + os.path.sep + shell_environment.GetEnvironment().set_shell_var("VS2019_PREFIX", prefix) + + # now confirm it exists + if not os.path.exists(shell_environment.GetEnvironment().get_shell_var("VS2019_PREFIX")): + self.Logger.error("Path for VS2019 toolchain is invalid") + return -2 + + return 0 + + def _get_vs_install_path(self, vs_version, varname): + # check if already specified + path = shell_environment.GetEnvironment().get_shell_var(varname) + if(path is None): + # Not specified...find latest + (rc, path) = FindWithVsWhere(vs_version=vs_version) + if rc == 0 and path is not None and os.path.exists(path): + self.Logger.debug("Found VS instance for %s", vs_version) + else: + self.Logger.error("Failed to find VS instance with VsWhere (%d)" % rc) + return path + + def _get_vc_version(self, path, varname): + # check if already specified + vc_ver = shell_environment.GetEnvironment().get_shell_var(varname) + if (path is None): + self.Logger.critical("Failed to find Visual Studio tools. Might need to check for VS install") + return vc_ver + if(vc_ver is None): + # Not specified...find latest + p2 = os.path.join(path, "VC", "Tools", "MSVC") + if not os.path.isdir(p2): + self.Logger.critical( + "Failed to find VC tools. Might need to check for VS install") + return vc_ver + vc_ver = os.listdir(p2)[-1].strip() # get last in list + self.Logger.debug("Found VC Tool version is %s" % vc_ver) + return vc_ver + + diff --git a/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain_plug_in.yaml b/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain_plug_in.yaml new file mode 100644 index 0000000000..30b8ec8766 --- /dev/null +++ b/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain_plug_in.yaml @@ -0,0 +1,11 @@ +## @file +# Build Plugin used to set the path to the visual studio tools chain +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "global-win", + "name": "Windows Visual Studio Tool Chain Support", + "module": "WindowsVsToolChain" +} -- 2.21.0.windows.1