From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mga12.intel.com (mga12.intel.com [192.55.52.136]) by mx.groups.io with SMTP id smtpd.web09.686.1572378925474782305 for ; Tue, 29 Oct 2019 12:55:25 -0700 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 192.55.52.136, mailfrom: michael.d.kinney@intel.com) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga001.fm.intel.com ([10.253.24.23]) by fmsmga106.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 29 Oct 2019 12:55:26 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.68,245,1569308400"; d="scan'208";a="211170213" Received: from mdkinney-mobl2.amr.corp.intel.com ([10.251.0.39]) by fmsmga001.fm.intel.com with ESMTP; 29 Oct 2019 12:55:25 -0700 From: "Michael D Kinney" To: devel@edk2.groups.io Cc: Sean Brogan , Bob Feng , Liming Gao Subject: [Patch v3 05/22] BaseTools: Add YAML files with path env and tool extdeps Date: Tue, 29 Oct 2019 12:55:00 -0700 Message-Id: <20191029195517.20028-6-michael.d.kinney@intel.com> X-Mailer: git-send-email 2.21.0.windows.1 In-Reply-To: <20191029195517.20028-1-michael.d.kinney@intel.com> References: <20191029195517.20028-1-michael.d.kinney@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit From: Sean Brogan https://bugzilla.tianocore.org/show_bug.cgi?id=2315 Cc: Bob Feng Cc: Liming Gao Signed-off-by: Michael D Kinney --- BaseTools/Bin/gcc_aarch64_linux_ext_dep.yaml | 21 +++ BaseTools/Bin/gcc_arm_linux_ext_dep.yaml | 21 +++ BaseTools/Bin/iasl_ext_dep.yaml | 21 +++ BaseTools/Bin/nasm_ext_dep.yaml | 18 ++ .../BinWrappers/PosixLike/posix_path_env.yaml | 10 ++ .../WindowsLike/win_build_tools_path_env.yaml | 10 ++ BaseTools/Edk2ToolsBuild.py | 163 ++++++++++++++++++ .../basetool_tiano_python_path_env.yaml | 11 ++ BaseTools/basetools_calling_path_env.yaml | 11 ++ BaseTools/basetools_path_env.yaml | 11 ++ 10 files changed, 297 insertions(+) create mode 100644 BaseTools/Bin/gcc_aarch64_linux_ext_dep.yaml create mode 100644 BaseTools/Bin/gcc_arm_linux_ext_dep.yaml create mode 100644 BaseTools/Bin/iasl_ext_dep.yaml create mode 100644 BaseTools/Bin/nasm_ext_dep.yaml create mode 100644 BaseTools/BinWrappers/PosixLike/posix_path_env.yaml create mode 100644 BaseTools/BinWrappers/WindowsLike/win_build_tools_path_env.yaml create mode 100644 BaseTools/Edk2ToolsBuild.py create mode 100644 BaseTools/Source/Python/basetool_tiano_python_path_env.yaml create mode 100644 BaseTools/basetools_calling_path_env.yaml create mode 100644 BaseTools/basetools_path_env.yaml diff --git a/BaseTools/Bin/gcc_aarch64_linux_ext_dep.yaml b/BaseTools/Bin/gcc_aarch64_linux_ext_dep.yaml new file mode 100644 index 0000000000..69648c89d8 --- /dev/null +++ b/BaseTools/Bin/gcc_aarch64_linux_ext_dep.yaml @@ -0,0 +1,21 @@ +## +# Download GCC AARCH64 compiler from Linaro's release site +# Set shell variable GCC5_AARCH64_INSTALL to this folder +# +# This is only downloaded when a build activates scope gcc_aarch64_linux +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "gcc_aarch64_linux", + "type": "web", + "name": "gcc_aarch64_linux", + "source": "http://releases.linaro.org/components/toolchain/binaries/7.4-2019.02/aarch64-linux-gnu/gcc-linaro-7.4.1-2019.02-x86_64_aarch64-linux-gnu.tar.xz", + "version": "7.4.1", + "sha256": "27f1dc2c491ed61ae8f0d4b0c11de59cd2f7dd9c94761ee7153006fcac1bf9ab", + "compression_type": "tar", + "internal_path": "/gcc-linaro-7.4.1-2019.02-x86_64_aarch64-linux-gnu/", + "flags": ["set_shell_var", ], + "var_name": "GCC5_AARCH64_INSTALL" +} diff --git a/BaseTools/Bin/gcc_arm_linux_ext_dep.yaml b/BaseTools/Bin/gcc_arm_linux_ext_dep.yaml new file mode 100644 index 0000000000..e108aaad2a --- /dev/null +++ b/BaseTools/Bin/gcc_arm_linux_ext_dep.yaml @@ -0,0 +1,21 @@ +## +# Download GCC ARM compiler from Linaro's release site +# Set shell variable GCC5_ARM_INSTALL to this folder +# +# This is only downloaded when a build activates scope gcc_arm_linux +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "gcc_arm_linux", + "type": "web", + "name": "gcc_arm_linux", + "source": "https://releases.linaro.org/components/toolchain/binaries/7.4-2019.02/arm-linux-gnueabihf/gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf.tar.xz", + "version": "7.4.1", + "sha256": "3C951CF1941D0FA06D64CC0D5E88612B209D8123B273FA26C16D70BD7BC6B163", + "compression_type": "tar", + "internal_path": "/gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf/", + "flags": ["set_shell_var", ], + "var_name": "GCC5_ARM_INSTALL" +} diff --git a/BaseTools/Bin/iasl_ext_dep.yaml b/BaseTools/Bin/iasl_ext_dep.yaml new file mode 100644 index 0000000000..1c63921367 --- /dev/null +++ b/BaseTools/Bin/iasl_ext_dep.yaml @@ -0,0 +1,21 @@ +## +# Download iasl executable tool from a nuget.org package +# - package contains different binaries based on host +# Add the folder with the tool to the path +# +# This is only downloaded for scope cibuild thus +# should have no impact on the asl compiler used by any +# given platform to build. +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "id": "iasl-ci-1", + "scope": "cibuild", + "type": "nuget", + "name": "iasl", + "source": "https://api.nuget.org/v3/index.json", + "version": "20190215.0.0", + "flags": ["set_path", "host_specific"] +} diff --git a/BaseTools/Bin/nasm_ext_dep.yaml b/BaseTools/Bin/nasm_ext_dep.yaml new file mode 100644 index 0000000000..8894516b5a --- /dev/null +++ b/BaseTools/Bin/nasm_ext_dep.yaml @@ -0,0 +1,18 @@ +## +# Download nasm x86 assembler executable tool from a nuget.org package +# - package contains different binaries based on host +# Put on the tool on the path +# +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "id": "nasm-1", + "scope": "edk2-build", + "type": "nuget", + "name": "mu_nasm", + "source": "https://api.nuget.org/v3/index.json", + "version": "2.14.02", + "flags": ["set_path", "host_specific"] +} diff --git a/BaseTools/BinWrappers/PosixLike/posix_path_env.yaml b/BaseTools/BinWrappers/PosixLike/posix_path_env.yaml new file mode 100644 index 0000000000..eec65ed7db --- /dev/null +++ b/BaseTools/BinWrappers/PosixLike/posix_path_env.yaml @@ -0,0 +1,10 @@ +## +# Set this folder on the path for all linux builds +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "global-nix", + "flags": ["set_path"] +} diff --git a/BaseTools/BinWrappers/WindowsLike/win_build_tools_path_env.yaml b/BaseTools/BinWrappers/WindowsLike/win_build_tools_path_env.yaml new file mode 100644 index 0000000000..340724f39f --- /dev/null +++ b/BaseTools/BinWrappers/WindowsLike/win_build_tools_path_env.yaml @@ -0,0 +1,10 @@ +## +# Add this folder to the path on Windows +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "global-win", + "flags": ["set_path"] +} diff --git a/BaseTools/Edk2ToolsBuild.py b/BaseTools/Edk2ToolsBuild.py new file mode 100644 index 0000000000..057d2e9e06 --- /dev/null +++ b/BaseTools/Edk2ToolsBuild.py @@ -0,0 +1,163 @@ +# @file Edk2ToolsBuild.py +# Invocable class that builds the basetool c files. +# +# Supports VS2017, VS2019, and GCC5 +## +# Copyright (c) Microsoft Corporation +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import os +import sys +import logging +import argparse +from edk2toolext import edk2_logging +from edk2toolext.environment import self_describing_environment +from edk2toolext.base_abstract_invocable import BaseAbstractInvocable +from edk2toollib.utility_functions import RunCmd +from edk2toollib.windows.locate_tools import QueryVcVariables + + +class Edk2ToolsBuild(BaseAbstractInvocable): + + def ParseCommandLineOptions(self): + ''' parse arguments ''' + ParserObj = argparse.ArgumentParser() + ParserObj.add_argument("-t", "--tool_chain_tag", dest="tct", default="VS2017", + help="Set the toolchain used to compile the build tools") + args = ParserObj.parse_args() + self.tool_chain_tag = args.tct + + def GetWorkspaceRoot(self): + ''' Return the workspace root for initializing the SDE ''' + + # this is the bastools dir...not the traditional EDK2 workspace root + return os.path.dirname(os.path.abspath(__file__)) + + def GetActiveScopes(self): + ''' return tuple containing scopes that should be active for this process ''' + + # for now don't use scopes + return ('global',) + + def GetLoggingLevel(self, loggerType): + ''' Get the logging level for a given type (return Logging.Level) + base == lowest logging level supported + con == Screen logging + txt == plain text file logging + md == markdown file logging + ''' + if(loggerType == "con"): + return logging.ERROR + else: + return logging.DEBUG + + def GetLoggingFolderRelativeToRoot(self): + ''' Return a path to folder for log files ''' + return "BaseToolsBuild" + + def GetVerifyCheckRequired(self): + ''' Will call self_describing_environment.VerifyEnvironment if this returns True ''' + return True + + def GetLoggingFileName(self, loggerType): + ''' Get the logging file name for the type. + Return None if the logger shouldn't be created + + base == lowest logging level supported + con == Screen logging + txt == plain text file logging + md == markdown file logging + ''' + return "BASETOOLS_BUILD" + + def WritePathEnvFile(self, OutputDir): + ''' Write a PyTool path env file for future PyTool based edk2 builds''' + content = '''## +# Set shell variable EDK_TOOLS_BIN to this folder +# +# Autogenerated by Edk2ToolsBuild.py +# +# Copyright (c), Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "id": "You-Built-BaseTools", + "scope": "edk2-build", + "flags": ["set_shell_var", "set_path"], + "var_name": "EDK_TOOLS_BIN" +} +''' + with open(os.path.join(OutputDir, "basetoolsbin_path_env.yaml"), "w") as f: + f.write(content) + + def Go(self): + logging.info("Running Python version: " + str(sys.version_info)) + + (build_env, shell_env) = self_describing_environment.BootstrapEnvironment( + self.GetWorkspaceRoot(), self.GetActiveScopes()) + + # # Bind our current execution environment into the shell vars. + ph = os.path.dirname(sys.executable) + if " " in ph: + ph = '"' + ph + '"' + shell_env.set_shell_var("PYTHON_HOME", ph) + # PYTHON_COMMAND is required to be set for using edk2 python builds. + pc = sys.executable + if " " in pc: + pc = '"' + pc + '"' + shell_env.set_shell_var("PYTHON_COMMAND", pc) + + if self.tool_chain_tag.lower().startswith("vs"): + + # # Update environment with required VC vars. + interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"] + interesting_keys.extend( + ["LIBPATH", "Path", "UniversalCRTSdkDir", "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath"]) + interesting_keys.extend( + ["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"]) + vc_vars = QueryVcVariables( + interesting_keys, 'x86', vs_version=self.tool_chain_tag.lower()) + for key in vc_vars.keys(): + logging.debug(f"Var - {key} = {vc_vars[key]}") + if key.lower() == 'path': + shell_env.insert_path(vc_vars[key]) + else: + shell_env.set_shell_var(key, vc_vars[key]) + + self.OutputDir = os.path.join( + shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32") + + # compiled tools need to be added to path because antlr is referenced + shell_env.insert_path(self.OutputDir) + + # Actually build the tools. + ret = RunCmd('nmake.exe', None, + workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH")) + if ret != 0: + raise Exception("Failed to build.") + + self.WritePathEnvFile(self.OutputDir) + return ret + + elif self.tool_chain_tag.lower().startswith("gcc"): + ret = RunCmd("make", "-C .", workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH")) + if ret != 0: + raise Exception("Failed to build.") + + self.OutputDir = os.path.join( + shell_env.get_shell_var("EDK_TOOLS_PATH"), "Source", "C", "bin") + + self.WritePathEnvFile(self.OutputDir) + return ret + + logging.critical("Tool Chain not supported") + return -1 + + +def main(): + Edk2ToolsBuild().Invoke() + + +if __name__ == "__main__": + main() diff --git a/BaseTools/Source/Python/basetool_tiano_python_path_env.yaml b/BaseTools/Source/Python/basetool_tiano_python_path_env.yaml new file mode 100644 index 0000000000..1edc8810ce --- /dev/null +++ b/BaseTools/Source/Python/basetool_tiano_python_path_env.yaml @@ -0,0 +1,11 @@ +## +# Add this folder to the pypath so modules can be easily +# loaded +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "global", + "flags": ["set_pypath"] +} diff --git a/BaseTools/basetools_calling_path_env.yaml b/BaseTools/basetools_calling_path_env.yaml new file mode 100644 index 0000000000..cfdbcea104 --- /dev/null +++ b/BaseTools/basetools_calling_path_env.yaml @@ -0,0 +1,11 @@ +## +# Set shell variable BASE_TOOL_PATH to this folder +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "global", + "flags": ["set_shell_var"], + "var_name": "BASE_TOOLS_PATH" +} diff --git a/BaseTools/basetools_path_env.yaml b/BaseTools/basetools_path_env.yaml new file mode 100644 index 0000000000..f64a3b8d45 --- /dev/null +++ b/BaseTools/basetools_path_env.yaml @@ -0,0 +1,11 @@ +## +# Set shell variable EDK_TOOL_PATH to this folder +# +# Copyright (c) 2019, Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "global", + "flags": ["set_shell_var", "set_build_var"], + "var_name": "EDK_TOOLS_PATH" +} -- 2.21.0.windows.1