[Patch v4 05/22] BaseTools: Add YAML files with path env and tool extdeps

["Michael D Kinney" <michael.d.kinney@intel.com>]

From: Sean Brogan <sean.brogan@microsoft.com>

Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Signed-off-by: Michael D Kinney <michael.d.kinney@intel.com>
---
 BaseTools/Bin/gcc_aarch64_linux_ext_dep.yaml  |  21 +++
 BaseTools/Bin/gcc_arm_linux_ext_dep.yaml      |  21 +++
 BaseTools/Bin/iasl_ext_dep.yaml               |  21 +++
 BaseTools/Bin/nasm_ext_dep.yaml               |  18 ++
 .../BinWrappers/PosixLike/posix_path_env.yaml |  10 ++
 .../WindowsLike/win_build_tools_path_env.yaml |  10 ++
 BaseTools/Edk2ToolsBuild.py                   | 163 ++++++++++++++++++
 .../basetool_tiano_python_path_env.yaml       |  11 ++
 BaseTools/basetools_calling_path_env.yaml     |  11 ++
 BaseTools/basetools_path_env.yaml             |  11 ++
 10 files changed, 297 insertions(+)
 create mode 100644 BaseTools/Bin/gcc_aarch64_linux_ext_dep.yaml
 create mode 100644 BaseTools/Bin/gcc_arm_linux_ext_dep.yaml
 create mode 100644 BaseTools/Bin/iasl_ext_dep.yaml
 create mode 100644 BaseTools/Bin/nasm_ext_dep.yaml
 create mode 100644 BaseTools/BinWrappers/PosixLike/posix_path_env.yaml
 create mode 100644 BaseTools/BinWrappers/WindowsLike/win_build_tools_path_env.yaml
 create mode 100644 BaseTools/Edk2ToolsBuild.py
 create mode 100644 BaseTools/Source/Python/basetool_tiano_python_path_env.yaml
 create mode 100644 BaseTools/basetools_calling_path_env.yaml
 create mode 100644 BaseTools/basetools_path_env.yaml

diff --git a/BaseTools/Bin/gcc_aarch64_linux_ext_dep.yaml b/BaseTools/Bin/gcc_aarch64_linux_ext_dep.yaml
new file mode 100644
index 0000000000..69648c89d8
--- /dev/null
+++ b/BaseTools/Bin/gcc_aarch64_linux_ext_dep.yaml
@@ -0,0 +1,21 @@
+##
+# Download GCC AARCH64 compiler from Linaro's release site
+# Set shell variable GCC5_AARCH64_INSTALL to this folder
+#
+# This is only downloaded when a build activates scope gcc_aarch64_linux
+#
+# Copyright (c) 2019, Microsoft Corporation
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+  "scope": "gcc_aarch64_linux",
+  "type": "web",
+  "name": "gcc_aarch64_linux",
+  "source": "http://releases.linaro.org/components/toolchain/binaries/7.4-2019.02/aarch64-linux-gnu/gcc-linaro-7.4.1-2019.02-x86_64_aarch64-linux-gnu.tar.xz",
+  "version": "7.4.1",
+  "sha256": "27f1dc2c491ed61ae8f0d4b0c11de59cd2f7dd9c94761ee7153006fcac1bf9ab",
+  "compression_type": "tar",
+  "internal_path": "/gcc-linaro-7.4.1-2019.02-x86_64_aarch64-linux-gnu/",
+  "flags": ["set_shell_var", ],
+  "var_name": "GCC5_AARCH64_INSTALL"
+}
diff --git a/BaseTools/Bin/gcc_arm_linux_ext_dep.yaml b/BaseTools/Bin/gcc_arm_linux_ext_dep.yaml
new file mode 100644
index 0000000000..e108aaad2a
--- /dev/null
+++ b/BaseTools/Bin/gcc_arm_linux_ext_dep.yaml
@@ -0,0 +1,21 @@
+##
+# Download GCC ARM compiler from Linaro's release site
+# Set shell variable GCC5_ARM_INSTALL to this folder
+#
+# This is only downloaded when a build activates scope gcc_arm_linux
+#
+# Copyright (c) 2019, Microsoft Corporation
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+  "scope": "gcc_arm_linux",
+  "type": "web",
+  "name": "gcc_arm_linux",
+  "source": "https://releases.linaro.org/components/toolchain/binaries/7.4-2019.02/arm-linux-gnueabihf/gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf.tar.xz",
+  "version": "7.4.1",
+  "sha256": "3C951CF1941D0FA06D64CC0D5E88612B209D8123B273FA26C16D70BD7BC6B163",
+  "compression_type": "tar",
+  "internal_path": "/gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf/",
+  "flags": ["set_shell_var", ],
+  "var_name": "GCC5_ARM_INSTALL"
+}
diff --git a/BaseTools/Bin/iasl_ext_dep.yaml b/BaseTools/Bin/iasl_ext_dep.yaml
new file mode 100644
index 0000000000..1c63921367
--- /dev/null
+++ b/BaseTools/Bin/iasl_ext_dep.yaml
@@ -0,0 +1,21 @@
+##
+# Download iasl executable tool from a nuget.org package
+# - package contains different binaries based on host
+# Add the folder with the tool to the path
+#
+# This is only downloaded for scope cibuild thus
+# should have no impact on the asl compiler used by any
+# given platform to build.
+#
+# Copyright (c) 2019, Microsoft Corporation
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+  "id": "iasl-ci-1",
+  "scope": "cibuild",
+  "type": "nuget",
+  "name": "iasl",
+  "source": "https://api.nuget.org/v3/index.json",
+  "version": "20190215.0.0",
+  "flags": ["set_path", "host_specific"]
+}
diff --git a/BaseTools/Bin/nasm_ext_dep.yaml b/BaseTools/Bin/nasm_ext_dep.yaml
new file mode 100644
index 0000000000..8894516b5a
--- /dev/null
+++ b/BaseTools/Bin/nasm_ext_dep.yaml
@@ -0,0 +1,18 @@
+##
+# Download nasm x86 assembler executable tool from a nuget.org package
+# - package contains different binaries based on host
+# Put on the tool on the path
+#
+#
+# Copyright (c) 2019, Microsoft Corporation
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+  "id": "nasm-1",
+  "scope": "edk2-build",
+  "type": "nuget",
+  "name": "mu_nasm",
+  "source": "https://api.nuget.org/v3/index.json",
+  "version": "2.14.02",
+  "flags": ["set_path", "host_specific"]
+}
diff --git a/BaseTools/BinWrappers/PosixLike/posix_path_env.yaml b/BaseTools/BinWrappers/PosixLike/posix_path_env.yaml
new file mode 100644
index 0000000000..eec65ed7db
--- /dev/null
+++ b/BaseTools/BinWrappers/PosixLike/posix_path_env.yaml
@@ -0,0 +1,10 @@
+##
+# Set this folder on the path for all linux builds
+#
+# Copyright (c) 2019, Microsoft Corporation
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+  "scope": "global-nix",
+  "flags": ["set_path"]
+}
diff --git a/BaseTools/BinWrappers/WindowsLike/win_build_tools_path_env.yaml b/BaseTools/BinWrappers/WindowsLike/win_build_tools_path_env.yaml
new file mode 100644
index 0000000000..340724f39f
--- /dev/null
+++ b/BaseTools/BinWrappers/WindowsLike/win_build_tools_path_env.yaml
@@ -0,0 +1,10 @@
+##
+# Add this folder to the path on Windows
+#
+# Copyright (c) 2019, Microsoft Corporation
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+  "scope": "global-win",
+  "flags": ["set_path"]
+}
diff --git a/BaseTools/Edk2ToolsBuild.py b/BaseTools/Edk2ToolsBuild.py
new file mode 100644
index 0000000000..057d2e9e06
--- /dev/null
+++ b/BaseTools/Edk2ToolsBuild.py
@@ -0,0 +1,163 @@
+# @file Edk2ToolsBuild.py
+# Invocable class that builds the basetool c files.
+#
+# Supports VS2017, VS2019, and GCC5
+##
+# Copyright (c) Microsoft Corporation
+#
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+import os
+import sys
+import logging
+import argparse
+from edk2toolext import edk2_logging
+from edk2toolext.environment import self_describing_environment
+from edk2toolext.base_abstract_invocable import BaseAbstractInvocable
+from edk2toollib.utility_functions import RunCmd
+from edk2toollib.windows.locate_tools import QueryVcVariables
+
+
+class Edk2ToolsBuild(BaseAbstractInvocable):
+
+    def ParseCommandLineOptions(self):
+        ''' parse arguments '''
+        ParserObj = argparse.ArgumentParser()
+        ParserObj.add_argument("-t", "--tool_chain_tag", dest="tct", default="VS2017",
+                               help="Set the toolchain used to compile the build tools")
+        args = ParserObj.parse_args()
+        self.tool_chain_tag = args.tct
+
+    def GetWorkspaceRoot(self):
+        ''' Return the workspace root for initializing the SDE '''
+
+        # this is the bastools dir...not the traditional EDK2 workspace root
+        return os.path.dirname(os.path.abspath(__file__))
+
+    def GetActiveScopes(self):
+        ''' return tuple containing scopes that should be active for this process '''
+
+        # for now don't use scopes
+        return ('global',)
+
+    def GetLoggingLevel(self, loggerType):
+        ''' Get the logging level for a given type (return Logging.Level)
+        base == lowest logging level supported
+        con  == Screen logging
+        txt  == plain text file logging
+        md   == markdown file logging
+        '''
+        if(loggerType == "con"):
+            return logging.ERROR
+        else:
+            return logging.DEBUG
+
+    def GetLoggingFolderRelativeToRoot(self):
+        ''' Return a path to folder for log files '''
+        return "BaseToolsBuild"
+
+    def GetVerifyCheckRequired(self):
+        ''' Will call self_describing_environment.VerifyEnvironment if this returns True '''
+        return True
+
+    def GetLoggingFileName(self, loggerType):
+        ''' Get the logging file name for the type.
+        Return None if the logger shouldn't be created
+
+        base == lowest logging level supported
+        con  == Screen logging
+        txt  == plain text file logging
+        md   == markdown file logging
+        '''
+        return "BASETOOLS_BUILD"
+
+    def WritePathEnvFile(self, OutputDir):
+        ''' Write a PyTool path env file for future PyTool based edk2 builds'''
+        content = '''##
+# Set shell variable EDK_TOOLS_BIN to this folder
+#
+# Autogenerated by Edk2ToolsBuild.py
+#
+# Copyright (c), Microsoft Corporation
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+  "id": "You-Built-BaseTools",
+  "scope": "edk2-build",
+  "flags": ["set_shell_var", "set_path"],
+  "var_name": "EDK_TOOLS_BIN"
+}
+'''
+        with open(os.path.join(OutputDir, "basetoolsbin_path_env.yaml"), "w") as f:
+            f.write(content)
+
+    def Go(self):
+        logging.info("Running Python version: " + str(sys.version_info))
+
+        (build_env, shell_env) = self_describing_environment.BootstrapEnvironment(
+            self.GetWorkspaceRoot(), self.GetActiveScopes())
+
+        # # Bind our current execution environment into the shell vars.
+        ph = os.path.dirname(sys.executable)
+        if " " in ph:
+            ph = '"' + ph + '"'
+        shell_env.set_shell_var("PYTHON_HOME", ph)
+        # PYTHON_COMMAND is required to be set for using edk2 python builds.
+        pc = sys.executable
+        if " " in pc:
+            pc = '"' + pc + '"'
+        shell_env.set_shell_var("PYTHON_COMMAND", pc)
+
+        if self.tool_chain_tag.lower().startswith("vs"):
+
+            # # Update environment with required VC vars.
+            interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"]
+            interesting_keys.extend(
+                ["LIBPATH", "Path", "UniversalCRTSdkDir", "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath"])
+            interesting_keys.extend(
+                ["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"])
+            vc_vars = QueryVcVariables(
+                interesting_keys, 'x86', vs_version=self.tool_chain_tag.lower())
+            for key in vc_vars.keys():
+                logging.debug(f"Var - {key} = {vc_vars[key]}")
+                if key.lower() == 'path':
+                    shell_env.insert_path(vc_vars[key])
+                else:
+                    shell_env.set_shell_var(key, vc_vars[key])
+
+            self.OutputDir = os.path.join(
+                shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")
+
+            # compiled tools need to be added to path because antlr is referenced
+            shell_env.insert_path(self.OutputDir)
+
+            # Actually build the tools.
+            ret = RunCmd('nmake.exe', None,
+                         workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
+            if ret != 0:
+                raise Exception("Failed to build.")
+
+            self.WritePathEnvFile(self.OutputDir)
+            return ret
+
+        elif self.tool_chain_tag.lower().startswith("gcc"):
+            ret = RunCmd("make", "-C .", workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
+            if ret != 0:
+                raise Exception("Failed to build.")
+
+            self.OutputDir = os.path.join(
+                shell_env.get_shell_var("EDK_TOOLS_PATH"), "Source", "C", "bin")
+
+            self.WritePathEnvFile(self.OutputDir)
+            return ret
+
+        logging.critical("Tool Chain not supported")
+        return -1
+
+
+def main():
+    Edk2ToolsBuild().Invoke()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/BaseTools/Source/Python/basetool_tiano_python_path_env.yaml b/BaseTools/Source/Python/basetool_tiano_python_path_env.yaml
new file mode 100644
index 0000000000..1edc8810ce
--- /dev/null
+++ b/BaseTools/Source/Python/basetool_tiano_python_path_env.yaml
@@ -0,0 +1,11 @@
+##
+# Add this folder to the pypath so modules can be easily
+# loaded
+#
+# Copyright (c) 2019, Microsoft Corporation
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+  "scope": "global",
+  "flags": ["set_pypath"]
+}
diff --git a/BaseTools/basetools_calling_path_env.yaml b/BaseTools/basetools_calling_path_env.yaml
new file mode 100644
index 0000000000..cfdbcea104
--- /dev/null
+++ b/BaseTools/basetools_calling_path_env.yaml
@@ -0,0 +1,11 @@
+##
+# Set shell variable BASE_TOOL_PATH to this folder
+#
+# Copyright (c) 2019, Microsoft Corporation
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+  "scope": "global",
+  "flags": ["set_shell_var"],
+  "var_name": "BASE_TOOLS_PATH"
+}
diff --git a/BaseTools/basetools_path_env.yaml b/BaseTools/basetools_path_env.yaml
new file mode 100644
index 0000000000..f64a3b8d45
--- /dev/null
+++ b/BaseTools/basetools_path_env.yaml
@@ -0,0 +1,11 @@
+##
+# Set shell variable EDK_TOOL_PATH to this folder
+#
+# Copyright (c) 2019, Microsoft Corporation
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+  "scope": "global",
+  "flags": ["set_shell_var", "set_build_var"],
+  "var_name": "EDK_TOOLS_PATH"
+}
-- 
2.21.0.windows.1