* [Patch 2/3] BaseTools: Enable --genfds-multi-thread to default build
2019-09-06 13:22 [Patch 0/3 V2]Enable --genfds-multi-thread to default build Bob Feng
2019-09-06 13:22 ` [Patch 1/3] BaseTools: Fixed the build fail on Linux with --genfds-multi-thread Bob Feng
@ 2019-09-06 13:22 ` Bob Feng
2019-09-06 13:22 ` [Patch 3/3] BaseTools: Fixed the bug of multi-thread genffs for override inf Bob Feng
2019-09-10 15:10 ` [edk2-devel] [Patch 0/3 V2]Enable --genfds-multi-thread to default build Liming Gao
3 siblings, 0 replies; 8+ messages in thread
From: Bob Feng @ 2019-09-06 13:22 UTC (permalink / raw)
To: devel; +Cc: Liming Gao, Bob Feng
BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1302
This patch enable --genfds-multi-thread to default build.
This patch keep --genfds-multi-thread build option for
compatibility and also add a new build option to disable
genfds-multi-thread as --no-genfds-multi-thread.
Cc: Liming Gao <liming.gao@intel.com>
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
---
BaseTools/Source/Python/AutoGen/GenMake.py | 4 ++--
BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py | 3 +--
BaseTools/Source/Python/Common/GlobalData.py | 2 +-
BaseTools/Source/Python/Common/buildoptions.py | 3 ++-
BaseTools/Source/Python/GenFds/GenFds.py | 9 ++++++---
BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py | 2 +-
BaseTools/Source/Python/build/build.py | 2 +-
7 files changed, 14 insertions(+), 11 deletions(-)
diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py
index 47dae82e1aeb..4f85a93055ab 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -1578,12 +1578,12 @@ class TopLevelMakefile(BuildFile):
elif LogLevel == EdkLogger.QUIET:
ExtraOption += " -q"
if GlobalData.gCaseInsensitive:
ExtraOption += " -c"
- if GlobalData.gEnableGenfdsMultiThread:
- ExtraOption += " --genfds-multi-thread"
+ if not GlobalData.gEnableGenfdsMultiThread:
+ ExtraOption += " --no-genfds-multi-thread"
if GlobalData.gIgnoreSource:
ExtraOption += " --ignore-sources"
for pcd in GlobalData.BuildOptionPcd:
if pcd[2]:
diff --git a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
index 24942674721f..365cfdefeaa1 100644
--- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
@@ -832,12 +832,11 @@ class WorkspaceAutoGen(AutoGen):
elif LogLevel <= EdkLogger.DEBUG_9:
FdsCommandDict["debug"] = LogLevel - 1
elif LogLevel == EdkLogger.QUIET:
FdsCommandDict["quiet"] = True
- if GlobalData.gEnableGenfdsMultiThread:
- FdsCommandDict["GenfdsMultiThread"] = True
+ FdsCommandDict["GenfdsMultiThread"] = GlobalData.gEnableGenfdsMultiThread
if GlobalData.gIgnoreSource:
FdsCommandDict["IgnoreSources"] = True
FdsCommandDict["OptionPcd"] = []
for pcd in GlobalData.BuildOptionPcd:
diff --git a/BaseTools/Source/Python/Common/GlobalData.py b/BaseTools/Source/Python/Common/GlobalData.py
index 61327ad8f10e..8eb72aa1d6f5 100755
--- a/BaseTools/Source/Python/Common/GlobalData.py
+++ b/BaseTools/Source/Python/Common/GlobalData.py
@@ -104,11 +104,11 @@ gUseHashCache = None
gBinCacheDest = None
gBinCacheSource = None
gPlatformHash = None
gPackageHash = {}
gModuleHash = {}
-gEnableGenfdsMultiThread = False
+gEnableGenfdsMultiThread = True
gSikpAutoGenCache = set()
# Dictionary for tracking Module build status as success or failure
# Top Dict: Key: Arch Type Value: Dictionary
# Second Dict: Key: AutoGen Obj Value: 'SUCCESS'\'FAIL'\'FAIL_METAFILE'
diff --git a/BaseTools/Source/Python/Common/buildoptions.py b/BaseTools/Source/Python/Common/buildoptions.py
index 7161aa66f23e..a717c58d8c9a 100644
--- a/BaseTools/Source/Python/Common/buildoptions.py
+++ b/BaseTools/Source/Python/Common/buildoptions.py
@@ -82,11 +82,12 @@ def MyOptionParser():
Parser.add_option("--pcd", action="append", dest="OptionPcd", help="Set PCD value by command line. Format: \"PcdName=Value\" ")
Parser.add_option("-l", "--cmd-len", action="store", type="int", dest="CommandLength", help="Specify the maximum line length of build command. Default is 4096.")
Parser.add_option("--hash", action="store_true", dest="UseHashCache", default=False, help="Enable hash-based caching during build process.")
Parser.add_option("--binary-destination", action="store", type="string", dest="BinCacheDest", help="Generate a cache of binary files in the specified directory.")
Parser.add_option("--binary-source", action="store", type="string", dest="BinCacheSource", help="Consume a cache of binary files from the specified directory.")
- Parser.add_option("--genfds-multi-thread", action="store_true", dest="GenfdsMultiThread", default=False, help="Enable GenFds multi thread to generate ffs file.")
+ Parser.add_option("--genfds-multi-thread", action="store_true", dest="GenfdsMultiThread", default=True, help="Enable GenFds multi thread to generate ffs file.")
+ Parser.add_option("--no-genfds-multi-thread", action="store_true", dest="NoGenfdsMultiThread", default=False, help="Disable GenFds multi thread to generate ffs file.")
Parser.add_option("--disable-include-path-check", action="store_true", dest="DisableIncludePathCheck", default=False, help="Disable the include path check for outside of package.")
(Opt, Args) = Parser.parse_args()
return (Opt, Args)
BuildOption, BuildTarget = MyOptionParser()
diff --git a/BaseTools/Source/Python/GenFds/GenFds.py b/BaseTools/Source/Python/GenFds/GenFds.py
index 51943411ad1f..d8bc28e4d0f8 100644
--- a/BaseTools/Source/Python/GenFds/GenFds.py
+++ b/BaseTools/Source/Python/GenFds/GenFds.py
@@ -91,11 +91,11 @@ def resetFdsGlobalVariable():
GenFdsGlobalVariable.GuidToolDefinition = {}
GenFdsGlobalVariable.FfsCmdDict = {}
GenFdsGlobalVariable.SecCmdList = []
GenFdsGlobalVariable.CopyList = []
GenFdsGlobalVariable.ModuleFile = ''
- GenFdsGlobalVariable.EnableGenfdsMultiThread = False
+ GenFdsGlobalVariable.EnableGenfdsMultiThread = True
GenFdsGlobalVariable.LargeFileInFvFlags = []
GenFdsGlobalVariable.EFI_FIRMWARE_FILE_SYSTEM3_GUID = '5473C07A-3DCB-4dca-BD6F-1E9689E7349A'
GenFdsGlobalVariable.LARGE_FILE_SIZE = 0x1000000
@@ -138,10 +138,12 @@ def GenFdsApi(FdsCommandDict, WorkSpaceDataBase=None):
GenFdsGlobalVariable.WorkSpaceDir = Workspace
if FdsCommandDict.get("debug"):
GenFdsGlobalVariable.VerboseLogger("Using Workspace:" + Workspace)
if FdsCommandDict.get("GenfdsMultiThread"):
GenFdsGlobalVariable.EnableGenfdsMultiThread = True
+ else:
+ GenFdsGlobalVariable.EnableGenfdsMultiThread = False
os.chdir(GenFdsGlobalVariable.WorkSpaceDir)
# set multiple workspace
PackagesPath = os.getenv("PACKAGES_PATH")
mws.setWs(GenFdsGlobalVariable.WorkSpaceDir, PackagesPath)
@@ -400,11 +402,11 @@ def OptionsToCommandDict(Options):
FdsCommandDict["verbose"] = Options.verbose
FdsCommandDict["FixedAddress"] = Options.FixedAddress
FdsCommandDict["quiet"] = Options.quiet
FdsCommandDict["debug"] = Options.debug
FdsCommandDict["Workspace"] = Options.Workspace
- FdsCommandDict["GenfdsMultiThread"] = Options.GenfdsMultiThread
+ FdsCommandDict["GenfdsMultiThread"] = not Options.NoGenfdsMultiThread
FdsCommandDict["fdf_file"] = [PathClass(Options.filename)] if Options.filename else []
FdsCommandDict["build_target"] = Options.BuildTarget
FdsCommandDict["toolchain_tag"] = Options.ToolChain
FdsCommandDict["active_platform"] = Options.activePlatform
FdsCommandDict["OptionPcd"] = Options.OptionPcd
@@ -457,11 +459,12 @@ def myOptionParser():
Parser.add_option("-D", "--define", action="append", type="string", dest="Macros", help="Macro: \"Name [= Value]\".")
Parser.add_option("-s", "--specifyaddress", dest="FixedAddress", action="store_true", type=None, help="Specify driver load address.")
Parser.add_option("--conf", action="store", type="string", dest="ConfDirectory", help="Specify the customized Conf directory.")
Parser.add_option("--ignore-sources", action="store_true", dest="IgnoreSources", default=False, help="Focus to a binary build and ignore all source files")
Parser.add_option("--pcd", action="append", dest="OptionPcd", help="Set PCD value by command line. Format: \"PcdName=Value\" ")
- Parser.add_option("--genfds-multi-thread", action="store_true", dest="GenfdsMultiThread", default=False, help="Enable GenFds multi thread to generate ffs file.")
+ Parser.add_option("--genfds-multi-thread", action="store_true", dest="GenfdsMultiThread", default=True, help="Enable GenFds multi thread to generate ffs file.")
+ Parser.add_option("--no-genfds-multi-thread", action="store_true", dest="NoGenfdsMultiThread", default=False, help="Disable GenFds multi thread to generate ffs file.")
Options, _ = Parser.parse_args()
return Options
## The class implementing the EDK2 flash image generation process
diff --git a/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py b/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
index 0f691ae52420..3dc73c8f61f9 100644
--- a/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
+++ b/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
@@ -67,11 +67,11 @@ class GenFdsGlobalVariable:
GuidToolDefinition = {}
FfsCmdDict = {}
SecCmdList = []
CopyList = []
ModuleFile = ''
- EnableGenfdsMultiThread = False
+ EnableGenfdsMultiThread = True
#
# The list whose element are flags to indicate if large FFS or SECTION files exist in FV.
# At the beginning of each generation of FV, false flag is appended to the list,
# after the call to GenerateSection returns, check the size of the output file,
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 0406ac314b65..2a10f99a1e70 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -726,11 +726,11 @@ class Build():
#Set global flag for build mode
GlobalData.gIgnoreSource = BuildOptions.IgnoreSources
GlobalData.gUseHashCache = BuildOptions.UseHashCache
GlobalData.gBinCacheDest = BuildOptions.BinCacheDest
GlobalData.gBinCacheSource = BuildOptions.BinCacheSource
- GlobalData.gEnableGenfdsMultiThread = BuildOptions.GenfdsMultiThread
+ GlobalData.gEnableGenfdsMultiThread = not BuildOptions.NoGenfdsMultiThread
GlobalData.gDisableIncludePathCheck = BuildOptions.DisableIncludePathCheck
if GlobalData.gBinCacheDest and not GlobalData.gUseHashCache:
EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination must be used together with --hash.")
--
2.20.1.windows.1
^ permalink raw reply related [flat|nested] 8+ messages in thread* [Patch 3/3] BaseTools: Fixed the bug of multi-thread genffs for override inf
2019-09-06 13:22 [Patch 0/3 V2]Enable --genfds-multi-thread to default build Bob Feng
2019-09-06 13:22 ` [Patch 1/3] BaseTools: Fixed the build fail on Linux with --genfds-multi-thread Bob Feng
2019-09-06 13:22 ` [Patch 2/3] BaseTools: Enable --genfds-multi-thread to default build Bob Feng
@ 2019-09-06 13:22 ` Bob Feng
2019-09-10 15:10 ` [edk2-devel] [Patch 0/3 V2]Enable --genfds-multi-thread to default build Liming Gao
3 siblings, 0 replies; 8+ messages in thread
From: Bob Feng @ 2019-09-06 13:22 UTC (permalink / raw)
To: devel; +Cc: Liming Gao, Steven Shi, Bob Feng
BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1302
If there is a inf override, and multi-thread genffs is enabled, build
will fail.
This patch is going to fix this bug.
Cc: Liming Gao <liming.gao@intel.com>
Cc: Steven Shi <steven.shi@intel.com>
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
---
BaseTools/Source/Python/AutoGen/AutoGenWorker.py | 2 +-
BaseTools/Source/Python/AutoGen/ModuleAutoGen.py | 2 +-
BaseTools/Source/Python/GenFds/FfsInfStatement.py | 5 ++++-
BaseTools/Source/Python/build/build.py | 8 ++++----
4 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
index 2e68538b1cb4..0d0ee319393c 100755
--- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
@@ -239,11 +239,11 @@ class AutoGenWorkerInProcess(mp.Process):
Ma.GenPreMakefileHash(GlobalData.gCacheIR)
if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
continue
Ma.CreateCodeFile(False)
- Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))
+ Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[]))
if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR)
Ma.GenMakeHash(GlobalData.gCacheIR)
if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR):
diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
index 3bb7e91154ac..8250b93fd440 100755
--- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
@@ -2121,11 +2121,11 @@ class ModuleAutoGen(AutoGen):
if (self.MetaFile.File,self.MetaFile.Root,self.Arch,self.MetaFile.Path) in GlobalData.Refes:
self.ReferenceModules = GlobalData.Refes[(self.MetaFile.File,self.MetaFile.Root,self.Arch,self.MetaFile.Path)]
self.CreateCodeFile()
if not (self.MetaFile.Path, self.Arch) in gDict or \
not gDict[(self.MetaFile.Path, self.Arch)].CreateMakeFileDone:
- self.CreateMakeFile(GenFfsList=GlobalData.FfsCmd.get((self.MetaFile.File, self.Arch),[]))
+ self.CreateMakeFile(GenFfsList=GlobalData.FfsCmd.get((self.MetaFile.Path, self.Arch),[]))
if not (self.MetaFile.Path, self.Arch) in gDict or \
not gDict[(self.MetaFile.Path, self.Arch)].CreateCodeFileDone or \
not gDict[(self.MetaFile.Path, self.Arch)].CreateMakeFileDone:
EdkLogger.quiet("[cache warning]: Cannot create CodeFile or Makefile for module %s[%s]" %(self.MetaFile.Path, self.Arch))
diff --git a/BaseTools/Source/Python/GenFds/FfsInfStatement.py b/BaseTools/Source/Python/GenFds/FfsInfStatement.py
index ebb93b14c4ff..43dbd0102f5e 100644
--- a/BaseTools/Source/Python/GenFds/FfsInfStatement.py
+++ b/BaseTools/Source/Python/GenFds/FfsInfStatement.py
@@ -500,11 +500,14 @@ class FfsInfStatement(FfsInfStatementClassObject):
#
MakefilePath = None
if self.IsBinaryModule:
IsMakefile = False
if IsMakefile:
- MakefilePath = self.InfFileName, Arch
+ PathClassObj = PathClass(self.InfFileName, GenFdsGlobalVariable.WorkSpaceDir)
+ if self.OverrideGuid:
+ PathClassObj = ProcessDuplicatedInf(PathClassObj, self.OverrideGuid, GenFdsGlobalVariable.WorkSpaceDir)
+ MakefilePath = PathClassObj.Path, Arch
if isinstance (Rule, RuleSimpleFile.RuleSimpleFile):
SectionOutputList = self.__GenSimpleFileSection__(Rule, IsMakefile=IsMakefile)
FfsOutput = self.__GenSimpleFileFfs__(Rule, SectionOutputList, MakefilePath=MakefilePath)
return FfsOutput
#
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 2a10f99a1e70..94318420196f 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -840,11 +840,11 @@ class Build():
PcdMa.GenPreMakefileHash(share_data)
if PcdMa.CanSkipbyPreMakefileCache(share_data):
continue
PcdMa.CreateCodeFile(False)
- PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[]))
+ PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[]))
if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
PcdMa.GenMakeHeaderFilesHash(share_data)
PcdMa.GenMakeHash(share_data)
if PcdMa.CanSkipbyMakeCache(share_data):
@@ -1867,13 +1867,13 @@ class Build():
self.Progress.Stop("done!")
if self.Target == "genc":
return True
if not self.SkipAutoGen or self.Target == 'genmake':
self.Progress.Start("Generating makefile")
- if CmdListDict and self.Fdf and (Module.File, Arch) in CmdListDict:
- Ma.CreateMakeFile(True, CmdListDict[Module.File, Arch])
- del CmdListDict[Module.File, Arch]
+ if CmdListDict and self.Fdf and (Module.Path, Arch) in CmdListDict:
+ Ma.CreateMakeFile(True, CmdListDict[Module.Path, Arch])
+ del CmdListDict[Module.Path, Arch]
else:
Ma.CreateMakeFile(True)
self.Progress.Stop("done!")
if self.Target == "genmake":
return True
--
2.20.1.windows.1
^ permalink raw reply related [flat|nested] 8+ messages in thread