[PATCH v1 0/3] Dynamic BaseTools Selection

[PATCH v1 0/3] Dynamic BaseTools Selection

[Joey Vagedes]

When using pytools to build, this patch adds the ability to dynamically
determine which BaseTools to build with. The Pypi BaseTools will be used
if present, otherwise defaulting to the in-tree Basetools.

Reference: https://bugzilla.tianocore.org/show_bug.cgi?id=4085

CC: Ard Biesheuvel <ardb+tianocore@kernel.org>
CC: Leif Lindholm <quic_llindhol@quicinc.com>
CC: Sami Mujawar <sami.mujawar@arm.com>
CC: Gerd Hoffmann <kraxel@redhat.com>
Cc: Andrew Fish <afish@apple.com>
Cc: Ray Ni <ray.ni@intel.com>

Signed-off-by: Joey Vagedes <Joeyvagedes@microsoft.com>

Joey Vagedes (3):
  ArmVirtPkg/PlatformCI: Dyn BaseTools Selection
  OvmfPkg/PlatformCI: Dyn BaseTools Selection
  EmulatorPkg/PlatformCI: Dyn BaseTools Selection

 ArmVirtPkg/PlatformCI/PlatformBuild.py  | 33 +++++++++++++++--
 EmulatorPkg/PlatformCI/PlatformBuild.py | 37 ++++++++++++++++++--
 OvmfPkg/PlatformCI/PlatformBuildLib.py  | 37 ++++++++++++++++++--
 3 files changed, 100 insertions(+), 7 deletions(-)

-- 
2.38.0.windows.1

[PATCH v1 1/3] ArmVirtPkg/PlatformCI: Dyn BaseTools Selection

[Joey Vagedes]

When using pytools to build ArmVirtPkg, add the ability to dynamically
determine which BaseTools to build with. The Pypi BaseTools will be used
if present, otherwise defaulting to the in-tree Basetools.

Reference: https://bugzilla.tianocore.org/show_bug.cgi?id=4085

CC: Ard Biesheuvel <ardb+tianocore@kernel.org>
CC: Leif Lindholm <quic_llindhol@quicinc.com>
CC: Sami Mujawar <sami.mujawar@arm.com>
CC: Gerd Hoffmann <kraxel@redhat.com>
Cc: Jiewen Yao <jiewen.yao@intel.com>
Cc: Jordan Justen <jordan.l.justen@intel.com>

Signed-off-by: Joey Vagedes <Joeyvagedes@microsoft.com>
---
 ArmVirtPkg/PlatformCI/PlatformBuild.py | 33 ++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/ArmVirtPkg/PlatformCI/PlatformBuild.py b/ArmVirtPkg/PlatformCI/PlatformBuild.py
index dff653e919..7ceca8145c 100644
--- a/ArmVirtPkg/PlatformCI/PlatformBuild.py
+++ b/ArmVirtPkg/PlatformCI/PlatformBuild.py
@@ -7,6 +7,7 @@
 import os
 import logging
 import io
+import importlib
 
 from edk2toolext.environment import shell_environment
 from edk2toolext.environment.uefi_build import UefiBuilder
@@ -40,6 +41,9 @@ class CommonPlatform():
 
 class SettingsManager(UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):
 
+    def __init__(self):
+        self.UseBuiltInBaseTools = None
+
     def GetPackagesSupported(self):
         ''' return iterable of edk2 packages supported by this build.
         These should be edk2 workspace relative paths '''
@@ -105,7 +109,18 @@ class SettingsManager(UpdateSettingsManager, SetupSettingsManager, PrEvalSetting
         scopes = CommonPlatform.Scopes
         ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")
 
-        if GetHostInfo().os.upper() == "LINUX" and ActualToolChainTag.upper().startswith("GCC"):
+        is_linux = GetHostInfo().os.upper() == "LINUX"
+
+        if self.UseBuiltInBaseTools is None:
+            if importlib.util.find_spec('edk2basetools') is not None:
+                self.UseBuiltInBaseTools = True
+            else:
+                self.UseBuiltInBaseTools = False
+
+        if self.UseBuiltInBaseTools is True:
+            scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)
+
+        if is_linux and ActualToolChainTag.upper().startswith("GCC"):
             if "AARCH64" in self.ActualArchitectures:
                 scopes += ("gcc_aarch64_linux",)
             if "ARM" in self.ActualArchitectures:
@@ -149,6 +164,7 @@ class SettingsManager(UpdateSettingsManager, SetupSettingsManager, PrEvalSetting
 
 class PlatformBuilder(UefiBuilder, BuildSettingsManager):
     def __init__(self):
+        self.UseBuiltInBaseTools = None
         UefiBuilder.__init__(self)
 
     def AddCommandLineOptions(self, parserObj):
@@ -179,7 +195,20 @@ class PlatformBuilder(UefiBuilder, BuildSettingsManager):
         ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")
         Arch = shell_environment.GetBuildVars().GetValue("TARGET_ARCH", "")
 
-        if GetHostInfo().os.upper() == "LINUX" and ActualToolChainTag.upper().startswith("GCC"):
+        is_linux = GetHostInfo().os.upper() == "LINUX"
+
+        if self.UseBuiltInBaseTools is None:
+            if importlib.util.find_spec('edk2basetools') is not None:
+                self.UseBuiltInBaseTools = True
+                logging.warning("Using Pip Tools based BaseTools")
+            else:
+                self.UseBuiltInBaseTools = False
+                logging.warning("Falling back to using in-tree BaseTools")
+
+        if self.UseBuiltInBaseTools is True:
+            scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)
+
+        if is_linux and ActualToolChainTag.upper().startswith("GCC"):
             if "AARCH64" == Arch:
                 scopes += ("gcc_aarch64_linux",)
             elif "ARM" == Arch:
-- 
2.38.0.windows.1

[PATCH v1 2/3] OvmfPkg/PlatformCI: Dyn BaseTools Selection

[Joey Vagedes]

When using pytools to build OvmfPkg, add the ability to dynamically
determine which BaseTools to build with. The Pypi BaseTools will be used
if present, otherwise defaulting to the in-tree Basetools.

Reference: https://bugzilla.tianocore.org/show_bug.cgi?id=4085

Cc: Ard Biesheuvel <ardb+tianocore@kernel.org>
Cc: Jiewen Yao <jiewen.yao@intel.com>
Cc: Jordan Justen <jordan.l.justen@intel.com>
Cc: Gerd Hoffmann <kraxel@redhat.com>

Signed-off-by: Joey Vagedes <Joeyvagedes@microsoft.com>
---
 OvmfPkg/PlatformCI/PlatformBuildLib.py | 37 ++++++++++++++++++--
 1 file changed, 34 insertions(+), 3 deletions(-)

diff --git a/OvmfPkg/PlatformCI/PlatformBuildLib.py b/OvmfPkg/PlatformCI/PlatformBuildLib.py
index bfef9849c7..d030216dff 100644
--- a/OvmfPkg/PlatformCI/PlatformBuildLib.py
+++ b/OvmfPkg/PlatformCI/PlatformBuildLib.py
@@ -7,6 +7,7 @@
 import os
 import logging
 import io
+import importlib
 
 from edk2toolext.environment import shell_environment
 from edk2toolext.environment.uefi_build import UefiBuilder
@@ -14,7 +15,7 @@ from edk2toolext.invocables.edk2_platform_build import BuildSettingsManager
 from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule
 from edk2toolext.invocables.edk2_update import UpdateSettingsManager
 from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager
-from edk2toollib.utility_functions import RunCmd
+from edk2toollib.utility_functions import RunCmd, GetHostInfo
 
 
     # ####################################################################################### #
@@ -22,6 +23,9 @@ from edk2toollib.utility_functions import RunCmd
     # ####################################################################################### #
 class SettingsManager(UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):
 
+    def __init__(self):
+        self.UseBuiltInBaseTools = None
+
     def GetPackagesSupported(self):
         ''' return iterable of edk2 packages supported by this build.
         These should be edk2 workspace relative paths '''
@@ -81,7 +85,19 @@ class SettingsManager(UpdateSettingsManager, SetupSettingsManager, PrEvalSetting
 
     def GetActiveScopes(self):
         ''' return tuple containing scopes that should be active for this process '''
-        return CommonPlatform.Scopes
+
+        scopes = CommonPlatform.Scopes
+        is_linux = GetHostInfo().os.upper() == "LINUX"
+        if self.UseBuiltInBaseTools is None:
+            if importlib.util.find_spec('edk2basetools') is not None:
+                self.UseBuiltInBaseTools = True
+            else:
+                self.UseBuiltInBaseTools = False
+
+        if self.UseBuiltInBaseTools is True:
+            scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)
+
+        return scopes
 
     def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:
         ''' Filter other cases that this package should be built
@@ -118,6 +134,7 @@ class SettingsManager(UpdateSettingsManager, SetupSettingsManager, PrEvalSetting
     # ####################################################################################### #
 class PlatformBuilder( UefiBuilder, BuildSettingsManager):
     def __init__(self):
+        self.UseBuiltInBaseTools = None
         UefiBuilder.__init__(self)
 
     def AddCommandLineOptions(self, parserObj):
@@ -144,7 +161,21 @@ class PlatformBuilder( UefiBuilder, BuildSettingsManager):
 
     def GetActiveScopes(self):
         ''' return tuple containing scopes that should be active for this process '''
-        return CommonPlatform.Scopes
+
+        scopes = CommonPlatform.Scopes
+        is_linux = GetHostInfo().os.upper() == "LINUX"
+        if self.UseBuiltInBaseTools is None:
+            if importlib.util.find_spec('edk2basetools') is not None:
+                self.UseBuiltInBaseTools = True
+                logging.warning("Using Pip Tools based BaseTools")
+            else:
+                self.UseBuiltInBaseTools = False
+                logging.warning("Falling back to using in-tree BaseTools")
+
+        if self.UseBuiltInBaseTools is True:
+            scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)
+
+        return scopes
 
     def GetName(self):
         ''' Get the name of the repo, platform, or product being build '''
-- 
2.38.0.windows.1

[PATCH v1 3/3] EmulatorPkg/PlatformCI: Dyn BaseTools Selection

[Joey Vagedes]

When using pytools to build EmulatorPkg, add the ability to dynamically
determine which BaseTools to build with. The Pypi BaseTools will be used
if present, otherwise defaulting to the in-tree Basetools.

Reference: https://bugzilla.tianocore.org/show_bug.cgi?id=4085

Cc: Andrew Fish <afish@apple.com>
Cc: Ray Ni <ray.ni@intel.com>

Signed-off-by: Joey Vagedes <Joeyvagedes@microsoft.com>
---
 EmulatorPkg/PlatformCI/PlatformBuild.py | 37 ++++++++++++++++++--
 1 file changed, 35 insertions(+), 2 deletions(-)

diff --git a/EmulatorPkg/PlatformCI/PlatformBuild.py b/EmulatorPkg/PlatformCI/PlatformBuild.py
index 6defbf6ef1..d6d37f6053 100644
--- a/EmulatorPkg/PlatformCI/PlatformBuild.py
+++ b/EmulatorPkg/PlatformCI/PlatformBuild.py
@@ -7,6 +7,7 @@
 import os
 import logging
 import io
+import importlib
 
 from edk2toolext.environment import shell_environment
 from edk2toolext.environment.uefi_build import UefiBuilder
@@ -40,6 +41,9 @@ class CommonPlatform():
 
 class SettingsManager(UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):
 
+    def __init__(self):
+        self.UseBuiltInBaseTools = None
+
     def GetPackagesSupported(self):
         ''' return iterable of edk2 packages supported by this build.
         These should be edk2 workspace relative paths '''
@@ -100,7 +104,20 @@ class SettingsManager(UpdateSettingsManager, SetupSettingsManager, PrEvalSetting
 
     def GetActiveScopes(self):
         ''' return tuple containing scopes that should be active for this process '''
-        return CommonPlatform.Scopes
+        scopes = CommonPlatform.Scopes
+
+        is_linux = GetHostInfo().os.upper() == "LINUX"
+
+        if self.UseBuiltInBaseTools is None:
+            if importlib.util.find_spec('edk2basetools') is not None:
+                self.UseBuiltInBaseTools = True
+            else:
+                self.UseBuiltInBaseTools = False
+
+        if self.UseBuiltInBaseTools is True:
+            scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)
+
+        return scopes
 
     def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:
         ''' Filter other cases that this package should be built
@@ -135,6 +152,7 @@ class SettingsManager(UpdateSettingsManager, SetupSettingsManager, PrEvalSetting
 
 class PlatformBuilder(UefiBuilder, BuildSettingsManager):
     def __init__(self):
+        self.UseBuiltInBaseTools = None
         UefiBuilder.__init__(self)
 
     def AddCommandLineOptions(self, parserObj):
@@ -161,7 +179,22 @@ class PlatformBuilder(UefiBuilder, BuildSettingsManager):
 
     def GetActiveScopes(self):
         ''' return tuple containing scopes that should be active for this process '''
-        return CommonPlatform.Scopes
+        scopes = CommonPlatform.Scopes
+
+        is_linux = GetHostInfo().os.upper() == "LINUX"
+
+        if self.UseBuiltInBaseTools is None:
+            if importlib.util.find_spec('edk2basetools') is not None:
+                self.UseBuiltInBaseTools = True
+                logging.warning("Using Pip Tools based BaseTools")
+            else:
+                self.UseBuiltInBaseTools = False
+                logging.warning("Falling back to using in-tree BaseTools")
+
+        if self.UseBuiltInBaseTools is True:
+            scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)
+
+        return scopes
 
     def GetName(self):
         ''' Get the name of the repo, platform, or product being build '''
-- 
2.38.0.windows.1