* [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen
@ 2019-09-05 9:04 Bob Feng
2019-09-05 13:11 ` [edk2-devel] " Liming Gao
0 siblings, 1 reply; 6+ messages in thread
From: Bob Feng @ 2019-09-05 9:04 UTC (permalink / raw)
To: devel; +Cc: Liming Gao, Steven Shi, Bob Feng
https://bugzilla.tianocore.org/show_bug.cgi?id=2080
This patch is to improve build -u option to re-use
GlobalVar_<platformguid>_<arch>.bin file which is
introduced by multiple-process-autogen feature.
Cc: Liming Gao <liming.gao@intel.com>
Cc: Steven Shi <steven.shi@intel.com>
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
---
.../Source/Python/AutoGen/AutoGenWorker.py | 2 +-
.../Source/Python/AutoGen/ModuleAutoGen.py | 4 +-
.../Python/AutoGen/ModuleAutoGenHelper.py | 29 ++
.../Source/Python/AutoGen/PlatformAutoGen.py | 2 +
.../Source/Python/AutoGen/WorkspaceAutoGen.py | 1 -
BaseTools/Source/Python/build/build.py | 355 +++++++++++-------
6 files changed, 255 insertions(+), 138 deletions(-)
diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
index 2e68538b1cb4..f488ae9d5f80 100755
--- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
@@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process):
Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
Ma.GenModuleFilesHash(GlobalData.gCacheIR)
Ma.GenPreMakefileHash(GlobalData.gCacheIR)
if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
- continue
+ continue
Ma.CreateCodeFile(False)
Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))
if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
index 3bb7e91154ac..5f28681e3146 100755
--- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
@@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen):
not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
self.GenModuleFilesHash(gDict)
if not (self.MetaFile.Path, self.Arch) in gDict or \
not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
- EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, self.Arch))
- return
+ EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, self.Arch))
+ return
# Initialze hash object
m = hashlib.md5()
# Add Platform level hash
diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
index c7591253debd..9dd93b9beb12 100644
--- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
+++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
@@ -234,22 +234,51 @@ class AutoGenInfo(object):
#
#This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in multiple thread
#
class WorkSpaceInfo(AutoGenInfo):
def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch):
+ if not hasattr(self, "_Init"):
+ self.do_init(Workspace, MetaFile, Target, ToolChain, Arch)
+ self._Init = True
+ def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch):
self._SrcTimeStamp = 0
self.Db = BuildDB
self.BuildDatabase = self.Db.BuildObject
self.Target = Target
self.ToolChain = ToolChain
self.WorkspaceDir = Workspace
self.ActivePlatform = MetaFile
self.ArchList = Arch
+ self.AutoGenObjectList = []
+ @property
+ def BuildDir(self):
+ return self.AutoGenObjectList[0].BuildDir
+ @property
+ def Name(self):
+ return self.AutoGenObjectList[0].Platform.PlatformName
+
+ @property
+ def FlashDefinition(self):
+ return self.AutoGenObjectList[0].Platform.FlashDefinition
+ @property
+ def GenFdsCommandDict(self):
+ FdsCommandDict = self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict")
+ if FdsCommandDict:
+ return FdsCommandDict
+ return {}
+
+ @cached_property
+ def FvDir(self):
+ return os.path.join(self.BuildDir, TAB_FV_DIRECTORY)
class PlatformInfo(AutoGenInfo):
def __init__(self, Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
+ if not hasattr(self, "_Init"):
+ self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe)
+ self._Init = True
+ def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
self.Wa = Workspace
self.WorkspaceDir = self.Wa.WorkspaceDir
self.MetaFile = MetaFile
self.Arch = Arch
self.Target = Target
diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
index 565424a95ead..1e17b6687129 100644
--- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
@@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen):
key = (Ma.MetaFile.File, self.Arch)
if key in FfsCommand:
Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key])
else:
Ma.CreateMakeFile(CreateModuleMakeFile)
+ self.CreateLibModuelDirs()
+ def CreateLibModuelDirs(self):
# no need to create makefile for the platform more than once
if self.IsMakeFileCreated:
return
# create library/module build dirs for platform
diff --git a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
index 24942674721f..f9d2c216169c 100644
--- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
@@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen):
self._CheckPcdDefineAndType()
self.CreateBuildOptionsFile()
self.CreatePcdTokenNumberFile()
self.CreateModuleHashInfo()
- GlobalData.gAutoGenPhase = False
#
# Merge Arch
#
def MergeArch(self):
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 0406ac314b65..db57c3e526da 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import PatchBinaryFile
import Common.GlobalData as GlobalData
from GenFds.GenFds import GenFds, GenFdsApi
import multiprocessing as mp
from multiprocessing import Manager
+from AutoGen.DataPipe import MemoryDataPipe
+from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo
+from GenFds.FdfParser import FdfParser
## standard targets of build command
gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
@@ -837,11 +840,11 @@ class Build():
for PcdMa in PcdMaList:
if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
PcdMa.GenModuleFilesHash(share_data)
PcdMa.GenPreMakefileHash(share_data)
if PcdMa.CanSkipbyPreMakefileCache(share_data):
- continue
+ continue
PcdMa.CreateCodeFile(False)
PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[]))
if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
@@ -1218,18 +1221,28 @@ class Build():
mqueue = mp.Queue()
for m in AutoGenObject.GetAllModuleInfo:
mqueue.put(m)
AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}
+ AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}
+ AutoGenObject.CreateLibModuelDirs()
+ AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
+ AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
+ AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": AutoGenObject.Workspace.GenFdsCommandDict}
self.Progress.Start("Generating makefile and code")
data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))
AutoGenObject.DataPipe.dump(data_pipe_file)
autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR)
+ AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
+ with open(AutoGenIdFile,"w") as fw:
+ fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList)))
+ fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir)
+ fw.write("PlatformGuid=%s\n" % str(AutoGenObject.Guid))
self.Progress.Stop("done!")
if not autogen_rt:
self.AutoGenMgr.TerminateWorkers()
- self.AutoGenMgr.join(0.1)
+ self.AutoGenMgr.join(1)
raise FatalError(errorcode)
AutoGenObject.CreateCodeFile(False)
AutoGenObject.CreateMakeFile(False)
else:
# always recreate top/platform makefile when clean, just in case of inconsistency
@@ -1778,10 +1791,11 @@ class Build():
self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
#
# Save MAP buffer into MAP file.
#
self._SaveMapFile (MapBuffer, Wa)
+ self.CreateGuidedSectionToolsFile(Wa)
## Build active module for different build targets, different tool chains and different archs
#
def _BuildModule(self):
for BuildTarget in self.BuildTargetList:
@@ -1852,13 +1866,13 @@ class Build():
if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
Ma.GenModuleFilesHash(GlobalData.gCacheIR)
Ma.GenPreMakefileHash(GlobalData.gCacheIR)
if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
- self.HashSkipModules.append(Ma)
- EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
- continue
+ self.HashSkipModules.append(Ma)
+ EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
+ continue
# Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
# for target which must generate AutoGen code and makefile
if not self.SkipAutoGen or self.Target == 'genc':
@@ -1987,125 +2001,221 @@ class Build():
GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData)
for Cmd in GenFfsDict:
tmpInf, tmpArch = GenFfsDict[Cmd]
CmdSetDict[tmpInf, tmpArch].add(Cmd)
return CmdSetDict
+ def VerifyAutoGenFiles(self):
+ AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
+ try:
+ with open(AutoGenIdFile) as fd:
+ lines = fd.readlines()
+ except:
+ return None
+ for line in lines:
+ if "Arch" in line:
+ ArchList = line.strip().split("=")[1].split("|")
+ if "BuildDir" in line:
+ BuildDir = line.split("=")[1].strip()
+ if "PlatformGuid" in line:
+ PlatformGuid = line.split("=")[1].strip()
+ GlobalVarList = []
+ for arch in ArchList:
+ global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch))
+ if not os.path.exists(global_var):
+ return None
+ GlobalVarList.append(global_var)
+ for global_var in GlobalVarList:
+ data_pipe = MemoryDataPipe()
+ data_pipe.load(global_var)
+ target = data_pipe.Get("P_Info").get("Target")
+ toolchain = data_pipe.Get("P_Info").get("ToolChain")
+ archlist = data_pipe.Get("P_Info").get("ArchList")
+ Arch = data_pipe.Get("P_Info").get("Arch")
+ active_p = data_pipe.Get("P_Info").get("ActivePlatform")
+ workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir")
+ PackagesPath = os.getenv("PACKAGES_PATH")
+ mws.setWs(workspacedir, PackagesPath)
+ LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList")
+ ModuleBuildDirectoryList = data_pipe.Get("ModuleBuildDirectoryList")
+
+ for m_build_dir in LibraryBuildDirectoryList:
+ if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
+ return None
+ for m_build_dir in ModuleBuildDirectoryList:
+ if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
+ return None
+ Wa = WorkSpaceInfo(
+ workspacedir,active_p,target,toolchain,archlist
+ )
+ Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe)
+ Wa.AutoGenObjectList.append(Pa)
+ return Wa
+ def SetupMakeSetting(self,Wa):
+ BuildModules = []
+ for Pa in Wa.AutoGenObjectList:
+ for m in Pa._MbList:
+ ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe)
+ BuildModules.append(ma)
+ fdf_file = Wa.FlashDefinition
+ if fdf_file:
+ Fdf = FdfParser(fdf_file.Path)
+ Fdf.ParseFile()
+ GlobalData.gFdfParser = Fdf
+ if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:
+ FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]
+ for FdRegion in FdDict.RegionList:
+ if str(FdRegion.RegionType) is 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):
+ if int(FdRegion.Offset) % 8 != 0:
+ EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' % (FdRegion.Offset))
+ Wa.FdfProfile = Fdf.Profile
+ self.Fdf = Fdf
+ else:
+ self.Fdf = None
+ return BuildModules
## Build a platform in multi-thread mode
#
- def _MultiThreadBuildPlatform(self):
- SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
- for BuildTarget in self.BuildTargetList:
- GlobalData.gGlobalDefines['TARGET'] = BuildTarget
- index = 0
- for ToolChain in self.ToolChainList:
- WorkspaceAutoGenTime = time.time()
- GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
- GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
- GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
- index += 1
- Wa = WorkspaceAutoGen(
- self.WorkspaceDir,
- self.PlatformFile,
- BuildTarget,
- ToolChain,
- self.ArchList,
- self.BuildDatabase,
- self.TargetTxt,
- self.ToolDef,
- self.Fdf,
- self.FdList,
- self.FvList,
- self.CapList,
- self.SkuId,
- self.UniFlag,
- self.Progress
- )
- self.Fdf = Wa.FdfFile
- self.LoadFixAddress = Wa.Platform.LoadFixAddress
- self.BuildReport.AddPlatformReport(Wa)
- Wa.CreateMakeFile(False)
+ def PerformAutoGen(self,BuildTarget,ToolChain):
+ WorkspaceAutoGenTime = time.time()
+ Wa = WorkspaceAutoGen(
+ self.WorkspaceDir,
+ self.PlatformFile,
+ BuildTarget,
+ ToolChain,
+ self.ArchList,
+ self.BuildDatabase,
+ self.TargetTxt,
+ self.ToolDef,
+ self.Fdf,
+ self.FdList,
+ self.FvList,
+ self.CapList,
+ self.SkuId,
+ self.UniFlag,
+ self.Progress
+ )
+ self.Fdf = Wa.FdfFile
+ self.LoadFixAddress = Wa.Platform.LoadFixAddress
+ self.BuildReport.AddPlatformReport(Wa)
+ Wa.CreateMakeFile(False)
# Add ffs build to makefile
- CmdListDict = {}
- if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
- CmdListDict = self._GenFfsCmd(Wa.ArchList)
+ CmdListDict = {}
+ if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
+ CmdListDict = self._GenFfsCmd(Wa.ArchList)
- # Add Platform and Package level hash in share_data for module hash calculation later
- if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
- GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
- for PkgName in GlobalData.gPackageHash.keys():
- GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]
+ # Add Platform and Package level hash in share_data for module hash calculation later
+ if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
+ GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
+ for PkgName in GlobalData.gPackageHash.keys():
+ GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]
- # multi-thread exit flag
- ExitFlag = threading.Event()
- ExitFlag.clear()
- self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
- self.BuildModules = []
- TotalModules = []
- for Arch in Wa.ArchList:
- PcdMaList = []
- AutoGenStart = time.time()
- GlobalData.gGlobalDefines['ARCH'] = Arch
- Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
- if Pa is None:
- continue
- ModuleList = []
- for Inf in Pa.Platform.Modules:
- ModuleList.append(Inf)
+ self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
+ BuildModules = []
+ TotalModules = []
+ for Arch in Wa.ArchList:
+ PcdMaList = []
+ AutoGenStart = time.time()
+ GlobalData.gGlobalDefines['ARCH'] = Arch
+ Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
+ if Pa is None:
+ continue
+ ModuleList = []
+ for Inf in Pa.Platform.Modules:
+ ModuleList.append(Inf)
# Add the INF only list in FDF
- if GlobalData.gFdfParser is not None:
- for InfName in GlobalData.gFdfParser.Profile.InfList:
- Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
- if Inf in Pa.Platform.Modules:
- continue
- ModuleList.append(Inf)
- Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
- Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
- Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
- for Module in ModuleList:
+ if GlobalData.gFdfParser is not None:
+ for InfName in GlobalData.gFdfParser.Profile.InfList:
+ Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
+ if Inf in Pa.Platform.Modules:
+ continue
+ ModuleList.append(Inf)
+ Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
+ Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
+ Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
+ Pa.CreateLibModuelDirs()
+ Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}
+ Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}
+ Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict}
+ ModuleCodaFile = {}
+ for ma in Pa.ModuleAutoGenList:
+ ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in ma.CodaTargetList]
+ Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}
+ for Module in ModuleList:
# Get ModuleAutoGen object to generate C code file and makefile
- Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
+ Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
- if Ma is None:
+ if Ma is None:
+ continue
+ if Ma.PcdIsDriver:
+ Ma.PlatformInfo = Pa
+ Ma.Workspace = Wa
+ PcdMaList.append(Ma)
+ TotalModules.append(Ma)
+ # Initialize all modules in tracking to 'FAIL'
+ GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
+
+
+ mqueue = mp.Queue()
+ for m in Pa.GetAllModuleInfo:
+ mqueue.put(m)
+ data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
+ Pa.DataPipe.dump(data_pipe_file)
+
+ autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR)
+
+ # Skip cache hit modules
+ if GlobalData.gBinCacheSource:
+ for Ma in TotalModules:
+ if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
+ GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:
+ self.HashSkipModules.append(Ma)
+ continue
+ if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
+ GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:
+ self.HashSkipModules.append(Ma)
continue
- if Ma.PcdIsDriver:
- Ma.PlatformInfo = Pa
- Ma.Workspace = Wa
- PcdMaList.append(Ma)
- TotalModules.append(Ma)
- # Initialize all modules in tracking to 'FAIL'
- GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
+ BuildModules.append(Ma)
+ else:
+ BuildModules.extend(TotalModules)
- mqueue = mp.Queue()
- for m in Pa.GetAllModuleInfo:
- mqueue.put(m)
- data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
- Pa.DataPipe.dump(data_pipe_file)
- autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR)
+ if not autogen_rt:
+ self.AutoGenMgr.TerminateWorkers()
+ self.AutoGenMgr.join(1)
+ raise FatalError(errorcode)
+ self.AutoGenTime += int(round((time.time() - AutoGenStart)))
+ AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
+ with open(AutoGenIdFile,"w") as fw:
+ fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))
+ fw.write("BuildDir=%s\n" % Wa.BuildDir)
+ fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid))
+ self.Progress.Stop("done!")
+ return Wa, BuildModules
- # Skip cache hit modules
- if GlobalData.gBinCacheSource:
- for Ma in TotalModules:
- if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
- GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:
- self.HashSkipModules.append(Ma)
- continue
- if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
- GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:
- self.HashSkipModules.append(Ma)
- continue
- self.BuildModules.append(Ma)
+ def _MultiThreadBuildPlatform(self):
+ SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
+ for BuildTarget in self.BuildTargetList:
+ GlobalData.gGlobalDefines['TARGET'] = BuildTarget
+ index = 0
+ for ToolChain in self.ToolChainList:
+ GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
+ GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
+ GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
+ index += 1
+ ExitFlag = threading.Event()
+ ExitFlag.clear()
+ if self.SkipAutoGen:
+ Wa = self.VerifyAutoGenFiles()
+ if Wa is None:
+ Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
else:
- self.BuildModules.extend(TotalModules)
-
- if not autogen_rt:
- self.AutoGenMgr.TerminateWorkers()
- self.AutoGenMgr.join(0.1)
- raise FatalError(errorcode)
- self.AutoGenTime += int(round((time.time() - AutoGenStart)))
- self.Progress.Stop("done!")
+ GlobalData.gAutoGenPhase = True
+ self.BuildModules = self.SetupMakeSetting(Wa)
+ else:
+ Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
+ Pa = Wa.AutoGenObjectList[0]
+ GlobalData.gAutoGenPhase = False
if GlobalData.gBinCacheSource:
EdkLogger.quiet("Total cache hit driver num: %s, cache miss driver num: %s" % (len(set(self.HashSkipModules)), len(set(self.BuildModules))))
CacheHitMa = set()
CacheNotHitMa = set()
@@ -2136,18 +2246,17 @@ class Build():
# Start task scheduler
if not BuildTask.IsOnGoing():
BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
# in case there's an interruption. we need a full version of makefile for platform
- Pa.CreateMakeFile(False)
+
if BuildTask.HasError():
self.invalidateHash()
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
self.MakeTime += int(round((time.time() - MakeStart)))
MakeContiue = time.time()
-
#
#
# All modules have been put in build tasks queue. Tell task scheduler
# to exit if all tasks are completed
#
@@ -2175,17 +2284,12 @@ class Build():
if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
#
# Get Module List
#
- ModuleList = {}
- for Pa in Wa.AutoGenObjectList:
- for Ma in Pa.ModuleAutoGenList:
- if Ma is None:
- continue
- if not Ma.IsLibrary:
- ModuleList[Ma.Guid.upper()] = Ma
+ ModuleList = {ma.Guid.upper():ma for ma in self.BuildModules}
+
#
# Rebase module to the preferred memory address before GenFds
#
MapBuffer = []
if self.LoadFixAddress != 0:
@@ -2206,33 +2310,17 @@ class Build():
self.GenFdsTime += int(round((time.time() - GenFdsStart)))
#
# Save MAP buffer into MAP file.
#
self._SaveMapFile(MapBuffer, Wa)
+ self.CreateGuidedSectionToolsFile(Wa)
self.invalidateHash()
-
## Generate GuidedSectionTools.txt in the FV directories.
#
- def CreateGuidedSectionToolsFile(self):
+ def CreateGuidedSectionToolsFile(self,Wa):
for BuildTarget in self.BuildTargetList:
for ToolChain in self.ToolChainList:
- Wa = WorkspaceAutoGen(
- self.WorkspaceDir,
- self.PlatformFile,
- BuildTarget,
- ToolChain,
- self.ArchList,
- self.BuildDatabase,
- self.TargetTxt,
- self.ToolDef,
- self.Fdf,
- self.FdList,
- self.FvList,
- self.CapList,
- self.SkuId,
- self.UniFlag
- )
FvDir = Wa.FvDir
if not os.path.exists(FvDir):
continue
for Arch in self.ArchList:
@@ -2285,11 +2373,10 @@ class Build():
if not self.SpawnMode or self.Target not in ["", "all"]:
self.SpawnMode = False
self._BuildPlatform()
else:
self._MultiThreadBuildPlatform()
- self.CreateGuidedSectionToolsFile()
else:
self.SpawnMode = False
self._BuildModule()
if self.Target == 'cleanall':
--
2.20.1.windows.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen
2019-09-05 9:04 [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen Bob Feng
@ 2019-09-05 13:11 ` Liming Gao
2019-09-05 14:04 ` Bob Feng
[not found] ` <15C18FCC38905317.10639@groups.io>
0 siblings, 2 replies; 6+ messages in thread
From: Liming Gao @ 2019-09-05 13:11 UTC (permalink / raw)
To: devel@edk2.groups.io, Feng, Bob C; +Cc: Shi, Steven
Bob:
Does this change only impact -u option behavior?
Thanks
Liming
> -----Original Message-----
> From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of Bob Feng
> Sent: Thursday, September 5, 2019 5:05 PM
> To: devel@edk2.groups.io
> Cc: Gao, Liming <liming.gao@intel.com>; Shi, Steven <steven.shi@intel.com>; Feng, Bob C <bob.c.feng@intel.com>
> Subject: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen
>
> https://bugzilla.tianocore.org/show_bug.cgi?id=2080
>
> This patch is to improve build -u option to re-use
> GlobalVar_<platformguid>_<arch>.bin file which is
> introduced by multiple-process-autogen feature.
>
> Cc: Liming Gao <liming.gao@intel.com>
> Cc: Steven Shi <steven.shi@intel.com>
> Signed-off-by: Bob Feng <bob.c.feng@intel.com>
> ---
> .../Source/Python/AutoGen/AutoGenWorker.py | 2 +-
> .../Source/Python/AutoGen/ModuleAutoGen.py | 4 +-
> .../Python/AutoGen/ModuleAutoGenHelper.py | 29 ++
> .../Source/Python/AutoGen/PlatformAutoGen.py | 2 +
> .../Source/Python/AutoGen/WorkspaceAutoGen.py | 1 -
> BaseTools/Source/Python/build/build.py | 355 +++++++++++-------
> 6 files changed, 255 insertions(+), 138 deletions(-)
>
> diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> index 2e68538b1cb4..f488ae9d5f80 100755
> --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process):
> Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
> if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
> Ma.GenModuleFilesHash(GlobalData.gCacheIR)
> Ma.GenPreMakefileHash(GlobalData.gCacheIR)
> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
> - continue
> + continue
>
> Ma.CreateCodeFile(False)
> Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))
>
> if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> index 3bb7e91154ac..5f28681e3146 100755
> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen):
> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
> self.GenModuleFilesHash(gDict)
>
> if not (self.MetaFile.Path, self.Arch) in gDict or \
> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
> - EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
> self.Arch))
> - return
> + EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
> self.Arch))
> + return
>
> # Initialze hash object
> m = hashlib.md5()
>
> # Add Platform level hash
> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> index c7591253debd..9dd93b9beb12 100644
> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> @@ -234,22 +234,51 @@ class AutoGenInfo(object):
> #
> #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in multiple thread
> #
> class WorkSpaceInfo(AutoGenInfo):
> def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch):
> + if not hasattr(self, "_Init"):
> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch)
> + self._Init = True
> + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch):
> self._SrcTimeStamp = 0
> self.Db = BuildDB
> self.BuildDatabase = self.Db.BuildObject
> self.Target = Target
> self.ToolChain = ToolChain
> self.WorkspaceDir = Workspace
> self.ActivePlatform = MetaFile
> self.ArchList = Arch
> + self.AutoGenObjectList = []
> + @property
> + def BuildDir(self):
> + return self.AutoGenObjectList[0].BuildDir
>
> + @property
> + def Name(self):
> + return self.AutoGenObjectList[0].Platform.PlatformName
> +
> + @property
> + def FlashDefinition(self):
> + return self.AutoGenObjectList[0].Platform.FlashDefinition
> + @property
> + def GenFdsCommandDict(self):
> + FdsCommandDict = self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict")
> + if FdsCommandDict:
> + return FdsCommandDict
> + return {}
> +
> + @cached_property
> + def FvDir(self):
> + return os.path.join(self.BuildDir, TAB_FV_DIRECTORY)
>
> class PlatformInfo(AutoGenInfo):
> def __init__(self, Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
> + if not hasattr(self, "_Init"):
> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe)
> + self._Init = True
> + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
> self.Wa = Workspace
> self.WorkspaceDir = self.Wa.WorkspaceDir
> self.MetaFile = MetaFile
> self.Arch = Arch
> self.Target = Target
> diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
> index 565424a95ead..1e17b6687129 100644
> --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
> +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
> @@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen):
> key = (Ma.MetaFile.File, self.Arch)
> if key in FfsCommand:
> Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key])
> else:
> Ma.CreateMakeFile(CreateModuleMakeFile)
> + self.CreateLibModuelDirs()
>
> + def CreateLibModuelDirs(self):
> # no need to create makefile for the platform more than once
> if self.IsMakeFileCreated:
> return
>
> # create library/module build dirs for platform
> diff --git a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
> index 24942674721f..f9d2c216169c 100644
> --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
> +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
> @@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen):
> self._CheckPcdDefineAndType()
>
> self.CreateBuildOptionsFile()
> self.CreatePcdTokenNumberFile()
> self.CreateModuleHashInfo()
> - GlobalData.gAutoGenPhase = False
>
> #
> # Merge Arch
> #
> def MergeArch(self):
> diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
> index 0406ac314b65..db57c3e526da 100755
> --- a/BaseTools/Source/Python/build/build.py
> +++ b/BaseTools/Source/Python/build/build.py
> @@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import PatchBinaryFile
>
> import Common.GlobalData as GlobalData
> from GenFds.GenFds import GenFds, GenFdsApi
> import multiprocessing as mp
> from multiprocessing import Manager
> +from AutoGen.DataPipe import MemoryDataPipe
> +from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo
> +from GenFds.FdfParser import FdfParser
>
>
> ## standard targets of build command
> gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
>
> @@ -837,11 +840,11 @@ class Build():
> for PcdMa in PcdMaList:
> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
> PcdMa.GenModuleFilesHash(share_data)
> PcdMa.GenPreMakefileHash(share_data)
> if PcdMa.CanSkipbyPreMakefileCache(share_data):
> - continue
> + continue
>
> PcdMa.CreateCodeFile(False)
> PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File,
> PcdMa.Arch),[]))
>
> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
> @@ -1218,18 +1221,28 @@ class Build():
> mqueue = mp.Queue()
> for m in AutoGenObject.GetAllModuleInfo:
> mqueue.put(m)
>
> AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}
> + AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}
> + AutoGenObject.CreateLibModuelDirs()
> + AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
> + AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
> + AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": AutoGenObject.Workspace.GenFdsCommandDict}
> self.Progress.Start("Generating makefile and code")
> data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" %
> (str(AutoGenObject.Guid),AutoGenObject.Arch))
> AutoGenObject.DataPipe.dump(data_pipe_file)
> autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList,
> GlobalData.gCacheIR)
> + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
> + with open(AutoGenIdFile,"w") as fw:
> + fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList)))
> + fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir)
> + fw.write("PlatformGuid=%s\n" % str(AutoGenObject.Guid))
> self.Progress.Stop("done!")
> if not autogen_rt:
> self.AutoGenMgr.TerminateWorkers()
> - self.AutoGenMgr.join(0.1)
> + self.AutoGenMgr.join(1)
> raise FatalError(errorcode)
> AutoGenObject.CreateCodeFile(False)
> AutoGenObject.CreateMakeFile(False)
> else:
> # always recreate top/platform makefile when clean, just in case of inconsistency
> @@ -1778,10 +1791,11 @@ class Build():
> self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
> #
> # Save MAP buffer into MAP file.
> #
> self._SaveMapFile (MapBuffer, Wa)
> + self.CreateGuidedSectionToolsFile(Wa)
>
> ## Build active module for different build targets, different tool chains and different archs
> #
> def _BuildModule(self):
> for BuildTarget in self.BuildTargetList:
> @@ -1852,13 +1866,13 @@ class Build():
>
> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
> Ma.GenModuleFilesHash(GlobalData.gCacheIR)
> Ma.GenPreMakefileHash(GlobalData.gCacheIR)
> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
> - self.HashSkipModules.append(Ma)
> - EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
> - continue
> + self.HashSkipModules.append(Ma)
> + EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
> + continue
>
> # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
> if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
> # for target which must generate AutoGen code and makefile
> if not self.SkipAutoGen or self.Target == 'genc':
> @@ -1987,125 +2001,221 @@ class Build():
> GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData)
> for Cmd in GenFfsDict:
> tmpInf, tmpArch = GenFfsDict[Cmd]
> CmdSetDict[tmpInf, tmpArch].add(Cmd)
> return CmdSetDict
> + def VerifyAutoGenFiles(self):
> + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
> + try:
> + with open(AutoGenIdFile) as fd:
> + lines = fd.readlines()
> + except:
> + return None
> + for line in lines:
> + if "Arch" in line:
> + ArchList = line.strip().split("=")[1].split("|")
> + if "BuildDir" in line:
> + BuildDir = line.split("=")[1].strip()
> + if "PlatformGuid" in line:
> + PlatformGuid = line.split("=")[1].strip()
> + GlobalVarList = []
> + for arch in ArchList:
> + global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch))
> + if not os.path.exists(global_var):
> + return None
> + GlobalVarList.append(global_var)
> + for global_var in GlobalVarList:
> + data_pipe = MemoryDataPipe()
> + data_pipe.load(global_var)
> + target = data_pipe.Get("P_Info").get("Target")
> + toolchain = data_pipe.Get("P_Info").get("ToolChain")
> + archlist = data_pipe.Get("P_Info").get("ArchList")
> + Arch = data_pipe.Get("P_Info").get("Arch")
> + active_p = data_pipe.Get("P_Info").get("ActivePlatform")
> + workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir")
> + PackagesPath = os.getenv("PACKAGES_PATH")
> + mws.setWs(workspacedir, PackagesPath)
> + LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList")
> + ModuleBuildDirectoryList = data_pipe.Get("ModuleBuildDirectoryList")
> +
> + for m_build_dir in LibraryBuildDirectoryList:
> + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
> + return None
> + for m_build_dir in ModuleBuildDirectoryList:
> + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
> + return None
> + Wa = WorkSpaceInfo(
> + workspacedir,active_p,target,toolchain,archlist
> + )
> + Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe)
> + Wa.AutoGenObjectList.append(Pa)
> + return Wa
> + def SetupMakeSetting(self,Wa):
> + BuildModules = []
> + for Pa in Wa.AutoGenObjectList:
> + for m in Pa._MbList:
> + ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe)
> + BuildModules.append(ma)
> + fdf_file = Wa.FlashDefinition
> + if fdf_file:
> + Fdf = FdfParser(fdf_file.Path)
> + Fdf.ParseFile()
> + GlobalData.gFdfParser = Fdf
> + if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:
> + FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]
> + for FdRegion in FdDict.RegionList:
> + if str(FdRegion.RegionType) is 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):
> + if int(FdRegion.Offset) % 8 != 0:
> + EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' %
> (FdRegion.Offset))
> + Wa.FdfProfile = Fdf.Profile
> + self.Fdf = Fdf
> + else:
> + self.Fdf = None
> + return BuildModules
>
> ## Build a platform in multi-thread mode
> #
> - def _MultiThreadBuildPlatform(self):
> - SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
> - for BuildTarget in self.BuildTargetList:
> - GlobalData.gGlobalDefines['TARGET'] = BuildTarget
> - index = 0
> - for ToolChain in self.ToolChainList:
> - WorkspaceAutoGenTime = time.time()
> - GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
> - GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
> - GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
> - index += 1
> - Wa = WorkspaceAutoGen(
> - self.WorkspaceDir,
> - self.PlatformFile,
> - BuildTarget,
> - ToolChain,
> - self.ArchList,
> - self.BuildDatabase,
> - self.TargetTxt,
> - self.ToolDef,
> - self.Fdf,
> - self.FdList,
> - self.FvList,
> - self.CapList,
> - self.SkuId,
> - self.UniFlag,
> - self.Progress
> - )
> - self.Fdf = Wa.FdfFile
> - self.LoadFixAddress = Wa.Platform.LoadFixAddress
> - self.BuildReport.AddPlatformReport(Wa)
> - Wa.CreateMakeFile(False)
> + def PerformAutoGen(self,BuildTarget,ToolChain):
> + WorkspaceAutoGenTime = time.time()
> + Wa = WorkspaceAutoGen(
> + self.WorkspaceDir,
> + self.PlatformFile,
> + BuildTarget,
> + ToolChain,
> + self.ArchList,
> + self.BuildDatabase,
> + self.TargetTxt,
> + self.ToolDef,
> + self.Fdf,
> + self.FdList,
> + self.FvList,
> + self.CapList,
> + self.SkuId,
> + self.UniFlag,
> + self.Progress
> + )
> + self.Fdf = Wa.FdfFile
> + self.LoadFixAddress = Wa.Platform.LoadFixAddress
> + self.BuildReport.AddPlatformReport(Wa)
> + Wa.CreateMakeFile(False)
>
> # Add ffs build to makefile
> - CmdListDict = {}
> - if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
> - CmdListDict = self._GenFfsCmd(Wa.ArchList)
> + CmdListDict = {}
> + if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
> + CmdListDict = self._GenFfsCmd(Wa.ArchList)
>
> - # Add Platform and Package level hash in share_data for module hash calculation later
> - if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
> - GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
> - for PkgName in GlobalData.gPackageHash.keys():
> - GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]
> + # Add Platform and Package level hash in share_data for module hash calculation later
> + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
> + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
> + for PkgName in GlobalData.gPackageHash.keys():
> + GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]
>
> - # multi-thread exit flag
> - ExitFlag = threading.Event()
> - ExitFlag.clear()
> - self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
> - self.BuildModules = []
> - TotalModules = []
> - for Arch in Wa.ArchList:
> - PcdMaList = []
> - AutoGenStart = time.time()
> - GlobalData.gGlobalDefines['ARCH'] = Arch
> - Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
> - if Pa is None:
> - continue
> - ModuleList = []
> - for Inf in Pa.Platform.Modules:
> - ModuleList.append(Inf)
> + self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
> + BuildModules = []
> + TotalModules = []
> + for Arch in Wa.ArchList:
> + PcdMaList = []
> + AutoGenStart = time.time()
> + GlobalData.gGlobalDefines['ARCH'] = Arch
> + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
> + if Pa is None:
> + continue
> + ModuleList = []
> + for Inf in Pa.Platform.Modules:
> + ModuleList.append(Inf)
> # Add the INF only list in FDF
> - if GlobalData.gFdfParser is not None:
> - for InfName in GlobalData.gFdfParser.Profile.InfList:
> - Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
> - if Inf in Pa.Platform.Modules:
> - continue
> - ModuleList.append(Inf)
> - Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
> - Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
> - Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
> - for Module in ModuleList:
> + if GlobalData.gFdfParser is not None:
> + for InfName in GlobalData.gFdfParser.Profile.InfList:
> + Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
> + if Inf in Pa.Platform.Modules:
> + continue
> + ModuleList.append(Inf)
> + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
> + Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
> + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
> + Pa.CreateLibModuelDirs()
> + Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}
> + Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}
> + Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict}
> + ModuleCodaFile = {}
> + for ma in Pa.ModuleAutoGenList:
> + ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in
> ma.CodaTargetList]
> + Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}
> + for Module in ModuleList:
> # Get ModuleAutoGen object to generate C code file and makefile
> - Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
> + Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
>
> - if Ma is None:
> + if Ma is None:
> + continue
> + if Ma.PcdIsDriver:
> + Ma.PlatformInfo = Pa
> + Ma.Workspace = Wa
> + PcdMaList.append(Ma)
> + TotalModules.append(Ma)
> + # Initialize all modules in tracking to 'FAIL'
> + GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
> +
> +
> + mqueue = mp.Queue()
> + for m in Pa.GetAllModuleInfo:
> + mqueue.put(m)
> + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
> + Pa.DataPipe.dump(data_pipe_file)
> +
> + autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR)
> +
> + # Skip cache hit modules
> + if GlobalData.gBinCacheSource:
> + for Ma in TotalModules:
> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
> + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:
> + self.HashSkipModules.append(Ma)
> + continue
> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
> + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:
> + self.HashSkipModules.append(Ma)
> continue
> - if Ma.PcdIsDriver:
> - Ma.PlatformInfo = Pa
> - Ma.Workspace = Wa
> - PcdMaList.append(Ma)
> - TotalModules.append(Ma)
> - # Initialize all modules in tracking to 'FAIL'
> - GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
> + BuildModules.append(Ma)
> + else:
> + BuildModules.extend(TotalModules)
>
> - mqueue = mp.Queue()
> - for m in Pa.GetAllModuleInfo:
> - mqueue.put(m)
> - data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
> - Pa.DataPipe.dump(data_pipe_file)
> - autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,
> GlobalData.gCacheIR)
> + if not autogen_rt:
> + self.AutoGenMgr.TerminateWorkers()
> + self.AutoGenMgr.join(1)
> + raise FatalError(errorcode)
> + self.AutoGenTime += int(round((time.time() - AutoGenStart)))
> + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
> + with open(AutoGenIdFile,"w") as fw:
> + fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))
> + fw.write("BuildDir=%s\n" % Wa.BuildDir)
> + fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid))
> + self.Progress.Stop("done!")
> + return Wa, BuildModules
>
> - # Skip cache hit modules
> - if GlobalData.gBinCacheSource:
> - for Ma in TotalModules:
> - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
> - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:
> - self.HashSkipModules.append(Ma)
> - continue
> - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
> - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:
> - self.HashSkipModules.append(Ma)
> - continue
> - self.BuildModules.append(Ma)
> + def _MultiThreadBuildPlatform(self):
> + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
> + for BuildTarget in self.BuildTargetList:
> + GlobalData.gGlobalDefines['TARGET'] = BuildTarget
> + index = 0
> + for ToolChain in self.ToolChainList:
> + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
> + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
> + GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
> + index += 1
> + ExitFlag = threading.Event()
> + ExitFlag.clear()
> + if self.SkipAutoGen:
> + Wa = self.VerifyAutoGenFiles()
> + if Wa is None:
> + Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
> else:
> - self.BuildModules.extend(TotalModules)
> -
> - if not autogen_rt:
> - self.AutoGenMgr.TerminateWorkers()
> - self.AutoGenMgr.join(0.1)
> - raise FatalError(errorcode)
> - self.AutoGenTime += int(round((time.time() - AutoGenStart)))
> - self.Progress.Stop("done!")
> + GlobalData.gAutoGenPhase = True
> + self.BuildModules = self.SetupMakeSetting(Wa)
> + else:
> + Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
> + Pa = Wa.AutoGenObjectList[0]
> + GlobalData.gAutoGenPhase = False
>
> if GlobalData.gBinCacheSource:
> EdkLogger.quiet("Total cache hit driver num: %s, cache miss driver num: %s" % (len(set(self.HashSkipModules)),
> len(set(self.BuildModules))))
> CacheHitMa = set()
> CacheNotHitMa = set()
> @@ -2136,18 +2246,17 @@ class Build():
> # Start task scheduler
> if not BuildTask.IsOnGoing():
> BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
>
> # in case there's an interruption. we need a full version of makefile for platform
> - Pa.CreateMakeFile(False)
> +
> if BuildTask.HasError():
> self.invalidateHash()
> EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
> self.MakeTime += int(round((time.time() - MakeStart)))
>
> MakeContiue = time.time()
> -
> #
> #
> # All modules have been put in build tasks queue. Tell task scheduler
> # to exit if all tasks are completed
> #
> @@ -2175,17 +2284,12 @@ class Build():
> if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and
> self.LoadFixAddress >= 0x100000000:
> EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to
> larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
> #
> # Get Module List
> #
> - ModuleList = {}
> - for Pa in Wa.AutoGenObjectList:
> - for Ma in Pa.ModuleAutoGenList:
> - if Ma is None:
> - continue
> - if not Ma.IsLibrary:
> - ModuleList[Ma.Guid.upper()] = Ma
> + ModuleList = {ma.Guid.upper():ma for ma in self.BuildModules}
> +
> #
> # Rebase module to the preferred memory address before GenFds
> #
> MapBuffer = []
> if self.LoadFixAddress != 0:
> @@ -2206,33 +2310,17 @@ class Build():
> self.GenFdsTime += int(round((time.time() - GenFdsStart)))
> #
> # Save MAP buffer into MAP file.
> #
> self._SaveMapFile(MapBuffer, Wa)
> + self.CreateGuidedSectionToolsFile(Wa)
> self.invalidateHash()
> -
> ## Generate GuidedSectionTools.txt in the FV directories.
> #
> - def CreateGuidedSectionToolsFile(self):
> + def CreateGuidedSectionToolsFile(self,Wa):
> for BuildTarget in self.BuildTargetList:
> for ToolChain in self.ToolChainList:
> - Wa = WorkspaceAutoGen(
> - self.WorkspaceDir,
> - self.PlatformFile,
> - BuildTarget,
> - ToolChain,
> - self.ArchList,
> - self.BuildDatabase,
> - self.TargetTxt,
> - self.ToolDef,
> - self.Fdf,
> - self.FdList,
> - self.FvList,
> - self.CapList,
> - self.SkuId,
> - self.UniFlag
> - )
> FvDir = Wa.FvDir
> if not os.path.exists(FvDir):
> continue
>
> for Arch in self.ArchList:
> @@ -2285,11 +2373,10 @@ class Build():
> if not self.SpawnMode or self.Target not in ["", "all"]:
> self.SpawnMode = False
> self._BuildPlatform()
> else:
> self._MultiThreadBuildPlatform()
> - self.CreateGuidedSectionToolsFile()
> else:
> self.SpawnMode = False
> self._BuildModule()
>
> if self.Target == 'cleanall':
> --
> 2.20.1.windows.1
>
>
>
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen
2019-09-05 13:11 ` [edk2-devel] " Liming Gao
@ 2019-09-05 14:04 ` Bob Feng
[not found] ` <15C18FCC38905317.10639@groups.io>
1 sibling, 0 replies; 6+ messages in thread
From: Bob Feng @ 2019-09-05 14:04 UTC (permalink / raw)
To: Gao, Liming, devel@edk2.groups.io; +Cc: Shi, Steven
Liming,
Yes, this change only impact -u option.
I separate _MultiThreadBuildPlatform function into two functions and create 2 new functions specially for IgnoreAutoGen, so the there looks big change for build.py in this patch. I verified this patch on Ovmf, E
-Bob
-----Original Message-----
From: Gao, Liming
Sent: Thursday, September 5, 2019 9:12 PM
To: devel@edk2.groups.io; Feng, Bob C <bob.c.feng@intel.com>
Cc: Shi, Steven <steven.shi@intel.com>
Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen
Bob:
Does this change only impact -u option behavior?
Thanks
Liming
> -----Original Message-----
> From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of
> Bob Feng
> Sent: Thursday, September 5, 2019 5:05 PM
> To: devel@edk2.groups.io
> Cc: Gao, Liming <liming.gao@intel.com>; Shi, Steven
> <steven.shi@intel.com>; Feng, Bob C <bob.c.feng@intel.com>
> Subject: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for
> IgnoreAutoGen
>
> https://bugzilla.tianocore.org/show_bug.cgi?id=2080
>
> This patch is to improve build -u option to re-use
> GlobalVar_<platformguid>_<arch>.bin file which is introduced by
> multiple-process-autogen feature.
>
> Cc: Liming Gao <liming.gao@intel.com>
> Cc: Steven Shi <steven.shi@intel.com>
> Signed-off-by: Bob Feng <bob.c.feng@intel.com>
> ---
> .../Source/Python/AutoGen/AutoGenWorker.py | 2 +-
> .../Source/Python/AutoGen/ModuleAutoGen.py | 4 +-
> .../Python/AutoGen/ModuleAutoGenHelper.py | 29 ++
> .../Source/Python/AutoGen/PlatformAutoGen.py | 2 +
> .../Source/Python/AutoGen/WorkspaceAutoGen.py | 1 -
> BaseTools/Source/Python/build/build.py | 355 +++++++++++-------
> 6 files changed, 255 insertions(+), 138 deletions(-)
>
> diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> index 2e68538b1cb4..f488ae9d5f80 100755
> --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process):
> Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
> if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
> Ma.GenModuleFilesHash(GlobalData.gCacheIR)
> Ma.GenPreMakefileHash(GlobalData.gCacheIR)
> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
> - continue
> + continue
>
> Ma.CreateCodeFile(False)
>
> Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File,
> Ma.Arch),[]))
>
> if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> index 3bb7e91154ac..5f28681e3146 100755
> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen):
> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
> self.GenModuleFilesHash(gDict)
>
> if not (self.MetaFile.Path, self.Arch) in gDict or \
> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
> - EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
> self.Arch))
> - return
> + EdkLogger.quiet("[cache warning]: Cannot generate
> + ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
> self.Arch))
> + return
>
> # Initialze hash object
> m = hashlib.md5()
>
> # Add Platform level hash
> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> index c7591253debd..9dd93b9beb12 100644
> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> @@ -234,22 +234,51 @@ class AutoGenInfo(object):
> #
> #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in
> multiple thread # class WorkSpaceInfo(AutoGenInfo):
> def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch):
> + if not hasattr(self, "_Init"):
> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch)
> + self._Init = True
> + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch):
> self._SrcTimeStamp = 0
> self.Db = BuildDB
> self.BuildDatabase = self.Db.BuildObject
> self.Target = Target
> self.ToolChain = ToolChain
> self.WorkspaceDir = Workspace
> self.ActivePlatform = MetaFile
> self.ArchList = Arch
> + self.AutoGenObjectList = []
> + @property
> + def BuildDir(self):
> + return self.AutoGenObjectList[0].BuildDir
>
> + @property
> + def Name(self):
> + return self.AutoGenObjectList[0].Platform.PlatformName
> +
> + @property
> + def FlashDefinition(self):
> + return self.AutoGenObjectList[0].Platform.FlashDefinition
> + @property
> + def GenFdsCommandDict(self):
> + FdsCommandDict = self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict")
> + if FdsCommandDict:
> + return FdsCommandDict
> + return {}
> +
> + @cached_property
> + def FvDir(self):
> + return os.path.join(self.BuildDir, TAB_FV_DIRECTORY)
>
> class PlatformInfo(AutoGenInfo):
> def __init__(self, Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
> + if not hasattr(self, "_Init"):
> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe)
> + self._Init = True
> + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
> self.Wa = Workspace
> self.WorkspaceDir = self.Wa.WorkspaceDir
> self.MetaFile = MetaFile
> self.Arch = Arch
> self.Target = Target
> diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
> b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
> index 565424a95ead..1e17b6687129 100644
> --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
> +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
> @@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen):
> key = (Ma.MetaFile.File, self.Arch)
> if key in FfsCommand:
> Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key])
> else:
> Ma.CreateMakeFile(CreateModuleMakeFile)
> + self.CreateLibModuelDirs()
>
> + def CreateLibModuelDirs(self):
> # no need to create makefile for the platform more than once
> if self.IsMakeFileCreated:
> return
>
> # create library/module build dirs for platform diff --git
> a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
> b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
> index 24942674721f..f9d2c216169c 100644
> --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
> +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
> @@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen):
> self._CheckPcdDefineAndType()
>
> self.CreateBuildOptionsFile()
> self.CreatePcdTokenNumberFile()
> self.CreateModuleHashInfo()
> - GlobalData.gAutoGenPhase = False
>
> #
> # Merge Arch
> #
> def MergeArch(self):
> diff --git a/BaseTools/Source/Python/build/build.py
> b/BaseTools/Source/Python/build/build.py
> index 0406ac314b65..db57c3e526da 100755
> --- a/BaseTools/Source/Python/build/build.py
> +++ b/BaseTools/Source/Python/build/build.py
> @@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import
> PatchBinaryFile
>
> import Common.GlobalData as GlobalData from GenFds.GenFds import
> GenFds, GenFdsApi import multiprocessing as mp from multiprocessing
> import Manager
> +from AutoGen.DataPipe import MemoryDataPipe from
> +AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo from
> +GenFds.FdfParser import FdfParser
>
>
> ## standard targets of build command
> gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries',
> 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
>
> @@ -837,11 +840,11 @@ class Build():
> for PcdMa in PcdMaList:
> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
> PcdMa.GenModuleFilesHash(share_data)
> PcdMa.GenPreMakefileHash(share_data)
> if PcdMa.CanSkipbyPreMakefileCache(share_data):
> - continue
> + continue
>
> PcdMa.CreateCodeFile(False)
> PcdMa.CreateMakeFile(False,GenFfsList =
> DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File,
> PcdMa.Arch),[]))
>
> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
> @@ -1218,18 +1221,28 @@ class Build():
> mqueue = mp.Queue()
> for m in AutoGenObject.GetAllModuleInfo:
> mqueue.put(m)
>
> AutoGenObject.DataPipe.DataContainer = {"CommandTarget":
> self.Target}
> + AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}
> + AutoGenObject.CreateLibModuelDirs()
> + AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
> + AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
> + AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict":
> + AutoGenObject.Workspace.GenFdsCommandDict}
> self.Progress.Start("Generating makefile and code")
> data_pipe_file = os.path.join(AutoGenObject.BuildDir,
> "GlobalVar_%s_%s.bin" %
> (str(AutoGenObject.Guid),AutoGenObject.Arch))
> AutoGenObject.DataPipe.dump(data_pipe_file)
> autogen_rt,errorcode = self.StartAutoGen(mqueue,
> AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList,
> GlobalData.gCacheIR)
> + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
> + with open(AutoGenIdFile,"w") as fw:
> + fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList)))
> + fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir)
> + fw.write("PlatformGuid=%s\n" %
> + str(AutoGenObject.Guid))
> self.Progress.Stop("done!")
> if not autogen_rt:
> self.AutoGenMgr.TerminateWorkers()
> - self.AutoGenMgr.join(0.1)
> + self.AutoGenMgr.join(1)
> raise FatalError(errorcode)
> AutoGenObject.CreateCodeFile(False)
> AutoGenObject.CreateMakeFile(False)
> else:
> # always recreate top/platform makefile when clean, just
> in case of inconsistency @@ -1778,10 +1791,11 @@ class Build():
> self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
> #
> # Save MAP buffer into MAP file.
> #
> self._SaveMapFile (MapBuffer, Wa)
> + self.CreateGuidedSectionToolsFile(Wa)
>
> ## Build active module for different build targets, different tool chains and different archs
> #
> def _BuildModule(self):
> for BuildTarget in self.BuildTargetList:
> @@ -1852,13 +1866,13 @@ class Build():
>
> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
> Ma.GenModuleFilesHash(GlobalData.gCacheIR)
> Ma.GenPreMakefileHash(GlobalData.gCacheIR)
> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
> - self.HashSkipModules.append(Ma)
> - EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
> - continue
> + self.HashSkipModules.append(Ma)
> + EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
> + continue
>
> # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
> if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
> # for target which must generate AutoGen code and makefile
> if not self.SkipAutoGen or self.Target == 'genc':
> @@ -1987,125 +2001,221 @@ class Build():
> GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData)
> for Cmd in GenFfsDict:
> tmpInf, tmpArch = GenFfsDict[Cmd]
> CmdSetDict[tmpInf, tmpArch].add(Cmd)
> return CmdSetDict
> + def VerifyAutoGenFiles(self):
> + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
> + try:
> + with open(AutoGenIdFile) as fd:
> + lines = fd.readlines()
> + except:
> + return None
> + for line in lines:
> + if "Arch" in line:
> + ArchList = line.strip().split("=")[1].split("|")
> + if "BuildDir" in line:
> + BuildDir = line.split("=")[1].strip()
> + if "PlatformGuid" in line:
> + PlatformGuid = line.split("=")[1].strip()
> + GlobalVarList = []
> + for arch in ArchList:
> + global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch))
> + if not os.path.exists(global_var):
> + return None
> + GlobalVarList.append(global_var)
> + for global_var in GlobalVarList:
> + data_pipe = MemoryDataPipe()
> + data_pipe.load(global_var)
> + target = data_pipe.Get("P_Info").get("Target")
> + toolchain = data_pipe.Get("P_Info").get("ToolChain")
> + archlist = data_pipe.Get("P_Info").get("ArchList")
> + Arch = data_pipe.Get("P_Info").get("Arch")
> + active_p = data_pipe.Get("P_Info").get("ActivePlatform")
> + workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir")
> + PackagesPath = os.getenv("PACKAGES_PATH")
> + mws.setWs(workspacedir, PackagesPath)
> + LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList")
> + ModuleBuildDirectoryList =
> + data_pipe.Get("ModuleBuildDirectoryList")
> +
> + for m_build_dir in LibraryBuildDirectoryList:
> + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
> + return None
> + for m_build_dir in ModuleBuildDirectoryList:
> + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
> + return None
> + Wa = WorkSpaceInfo(
> + workspacedir,active_p,target,toolchain,archlist
> + )
> + Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe)
> + Wa.AutoGenObjectList.append(Pa)
> + return Wa
> + def SetupMakeSetting(self,Wa):
> + BuildModules = []
> + for Pa in Wa.AutoGenObjectList:
> + for m in Pa._MbList:
> + ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe)
> + BuildModules.append(ma)
> + fdf_file = Wa.FlashDefinition
> + if fdf_file:
> + Fdf = FdfParser(fdf_file.Path)
> + Fdf.ParseFile()
> + GlobalData.gFdfParser = Fdf
> + if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:
> + FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]
> + for FdRegion in FdDict.RegionList:
> + if str(FdRegion.RegionType) is 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):
> + if int(FdRegion.Offset) % 8 != 0:
> + EdkLogger.error("build", FORMAT_INVALID,
> + 'The VPD Base Address %s must be 8-byte aligned.' %
> (FdRegion.Offset))
> + Wa.FdfProfile = Fdf.Profile
> + self.Fdf = Fdf
> + else:
> + self.Fdf = None
> + return BuildModules
>
> ## Build a platform in multi-thread mode
> #
> - def _MultiThreadBuildPlatform(self):
> - SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
> - for BuildTarget in self.BuildTargetList:
> - GlobalData.gGlobalDefines['TARGET'] = BuildTarget
> - index = 0
> - for ToolChain in self.ToolChainList:
> - WorkspaceAutoGenTime = time.time()
> - GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
> - GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
> - GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
> - index += 1
> - Wa = WorkspaceAutoGen(
> - self.WorkspaceDir,
> - self.PlatformFile,
> - BuildTarget,
> - ToolChain,
> - self.ArchList,
> - self.BuildDatabase,
> - self.TargetTxt,
> - self.ToolDef,
> - self.Fdf,
> - self.FdList,
> - self.FvList,
> - self.CapList,
> - self.SkuId,
> - self.UniFlag,
> - self.Progress
> - )
> - self.Fdf = Wa.FdfFile
> - self.LoadFixAddress = Wa.Platform.LoadFixAddress
> - self.BuildReport.AddPlatformReport(Wa)
> - Wa.CreateMakeFile(False)
> + def PerformAutoGen(self,BuildTarget,ToolChain):
> + WorkspaceAutoGenTime = time.time()
> + Wa = WorkspaceAutoGen(
> + self.WorkspaceDir,
> + self.PlatformFile,
> + BuildTarget,
> + ToolChain,
> + self.ArchList,
> + self.BuildDatabase,
> + self.TargetTxt,
> + self.ToolDef,
> + self.Fdf,
> + self.FdList,
> + self.FvList,
> + self.CapList,
> + self.SkuId,
> + self.UniFlag,
> + self.Progress
> + )
> + self.Fdf = Wa.FdfFile
> + self.LoadFixAddress = Wa.Platform.LoadFixAddress
> + self.BuildReport.AddPlatformReport(Wa)
> + Wa.CreateMakeFile(False)
>
> # Add ffs build to makefile
> - CmdListDict = {}
> - if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
> - CmdListDict = self._GenFfsCmd(Wa.ArchList)
> + CmdListDict = {}
> + if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
> + CmdListDict = self._GenFfsCmd(Wa.ArchList)
>
> - # Add Platform and Package level hash in share_data for module hash calculation later
> - if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
> - GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
> - for PkgName in GlobalData.gPackageHash.keys():
> - GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]
> + # Add Platform and Package level hash in share_data for module hash calculation later
> + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
> + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
> + for PkgName in GlobalData.gPackageHash.keys():
> + GlobalData.gCacheIR[(PkgName, 'PackageHash')] =
> + GlobalData.gPackageHash[PkgName]
>
> - # multi-thread exit flag
> - ExitFlag = threading.Event()
> - ExitFlag.clear()
> - self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
> - self.BuildModules = []
> - TotalModules = []
> - for Arch in Wa.ArchList:
> - PcdMaList = []
> - AutoGenStart = time.time()
> - GlobalData.gGlobalDefines['ARCH'] = Arch
> - Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
> - if Pa is None:
> - continue
> - ModuleList = []
> - for Inf in Pa.Platform.Modules:
> - ModuleList.append(Inf)
> + self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
> + BuildModules = []
> + TotalModules = []
> + for Arch in Wa.ArchList:
> + PcdMaList = []
> + AutoGenStart = time.time()
> + GlobalData.gGlobalDefines['ARCH'] = Arch
> + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
> + if Pa is None:
> + continue
> + ModuleList = []
> + for Inf in Pa.Platform.Modules:
> + ModuleList.append(Inf)
> # Add the INF only list in FDF
> - if GlobalData.gFdfParser is not None:
> - for InfName in GlobalData.gFdfParser.Profile.InfList:
> - Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
> - if Inf in Pa.Platform.Modules:
> - continue
> - ModuleList.append(Inf)
> - Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
> - Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
> - Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
> - for Module in ModuleList:
> + if GlobalData.gFdfParser is not None:
> + for InfName in GlobalData.gFdfParser.Profile.InfList:
> + Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
> + if Inf in Pa.Platform.Modules:
> + continue
> + ModuleList.append(Inf)
> + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
> + Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
> + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
> + Pa.CreateLibModuelDirs()
> + Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}
> + Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}
> + Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict}
> + ModuleCodaFile = {}
> + for ma in Pa.ModuleAutoGenList:
> +
> + ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFil
> + e.Path)] = [item.Target for item in
> ma.CodaTargetList]
> + Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}
> + for Module in ModuleList:
> # Get ModuleAutoGen object to generate C code file and makefile
> - Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
> + Ma = ModuleAutoGen(Wa, Module, BuildTarget,
> + ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
>
> - if Ma is None:
> + if Ma is None:
> + continue
> + if Ma.PcdIsDriver:
> + Ma.PlatformInfo = Pa
> + Ma.Workspace = Wa
> + PcdMaList.append(Ma)
> + TotalModules.append(Ma)
> + # Initialize all modules in tracking to 'FAIL'
> + GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
> +
> +
> + mqueue = mp.Queue()
> + for m in Pa.GetAllModuleInfo:
> + mqueue.put(m)
> + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
> + Pa.DataPipe.dump(data_pipe_file)
> +
> + autogen_rt, errorcode = self.StartAutoGen(mqueue,
> + Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR)
> +
> + # Skip cache hit modules
> + if GlobalData.gBinCacheSource:
> + for Ma in TotalModules:
> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
> + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:
> + self.HashSkipModules.append(Ma)
> + continue
> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
> + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:
> + self.HashSkipModules.append(Ma)
> continue
> - if Ma.PcdIsDriver:
> - Ma.PlatformInfo = Pa
> - Ma.Workspace = Wa
> - PcdMaList.append(Ma)
> - TotalModules.append(Ma)
> - # Initialize all modules in tracking to 'FAIL'
> - GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
> + BuildModules.append(Ma)
> + else:
> + BuildModules.extend(TotalModules)
>
> - mqueue = mp.Queue()
> - for m in Pa.GetAllModuleInfo:
> - mqueue.put(m)
> - data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
> - Pa.DataPipe.dump(data_pipe_file)
> - autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,
> GlobalData.gCacheIR)
> + if not autogen_rt:
> + self.AutoGenMgr.TerminateWorkers()
> + self.AutoGenMgr.join(1)
> + raise FatalError(errorcode)
> + self.AutoGenTime += int(round((time.time() - AutoGenStart)))
> + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
> + with open(AutoGenIdFile,"w") as fw:
> + fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))
> + fw.write("BuildDir=%s\n" % Wa.BuildDir)
> + fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid))
> + self.Progress.Stop("done!")
> + return Wa, BuildModules
>
> - # Skip cache hit modules
> - if GlobalData.gBinCacheSource:
> - for Ma in TotalModules:
> - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
> - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:
> - self.HashSkipModules.append(Ma)
> - continue
> - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
> - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:
> - self.HashSkipModules.append(Ma)
> - continue
> - self.BuildModules.append(Ma)
> + def _MultiThreadBuildPlatform(self):
> + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
> + for BuildTarget in self.BuildTargetList:
> + GlobalData.gGlobalDefines['TARGET'] = BuildTarget
> + index = 0
> + for ToolChain in self.ToolChainList:
> + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
> + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
> + GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
> + index += 1
> + ExitFlag = threading.Event()
> + ExitFlag.clear()
> + if self.SkipAutoGen:
> + Wa = self.VerifyAutoGenFiles()
> + if Wa is None:
> + Wa, self.BuildModules =
> + self.PerformAutoGen(BuildTarget,ToolChain)
> else:
> - self.BuildModules.extend(TotalModules)
> -
> - if not autogen_rt:
> - self.AutoGenMgr.TerminateWorkers()
> - self.AutoGenMgr.join(0.1)
> - raise FatalError(errorcode)
> - self.AutoGenTime += int(round((time.time() - AutoGenStart)))
> - self.Progress.Stop("done!")
> + GlobalData.gAutoGenPhase = True
> + self.BuildModules = self.SetupMakeSetting(Wa)
> + else:
> + Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
> + Pa = Wa.AutoGenObjectList[0]
> + GlobalData.gAutoGenPhase = False
>
> if GlobalData.gBinCacheSource:
> EdkLogger.quiet("Total cache hit driver num: %s,
> cache miss driver num: %s" % (len(set(self.HashSkipModules)),
> len(set(self.BuildModules))))
> CacheHitMa = set()
> CacheNotHitMa = set() @@ -2136,18 +2246,17 @@
> class Build():
> # Start task scheduler
> if not BuildTask.IsOnGoing():
>
> BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
>
> # in case there's an interruption. we need a full version of makefile for platform
> - Pa.CreateMakeFile(False)
> +
> if BuildTask.HasError():
> self.invalidateHash()
> EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
> self.MakeTime += int(round((time.time() -
> MakeStart)))
>
> MakeContiue = time.time()
> -
> #
> #
> # All modules have been put in build tasks queue. Tell task scheduler
> # to exit if all tasks are completed
> #
> @@ -2175,17 +2284,12 @@ class Build():
> if (Arch == 'IA32' or Arch == 'ARM') and
> self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
> EdkLogger.error("build",
> PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
> #
> # Get Module List
> #
> - ModuleList = {}
> - for Pa in Wa.AutoGenObjectList:
> - for Ma in Pa.ModuleAutoGenList:
> - if Ma is None:
> - continue
> - if not Ma.IsLibrary:
> - ModuleList[Ma.Guid.upper()] = Ma
> + ModuleList = {ma.Guid.upper():ma for ma in
> + self.BuildModules}
> +
> #
> # Rebase module to the preferred memory address before GenFds
> #
> MapBuffer = []
> if self.LoadFixAddress != 0:
> @@ -2206,33 +2310,17 @@ class Build():
> self.GenFdsTime += int(round((time.time() - GenFdsStart)))
> #
> # Save MAP buffer into MAP file.
> #
> self._SaveMapFile(MapBuffer, Wa)
> + self.CreateGuidedSectionToolsFile(Wa)
> self.invalidateHash()
> -
> ## Generate GuidedSectionTools.txt in the FV directories.
> #
> - def CreateGuidedSectionToolsFile(self):
> + def CreateGuidedSectionToolsFile(self,Wa):
> for BuildTarget in self.BuildTargetList:
> for ToolChain in self.ToolChainList:
> - Wa = WorkspaceAutoGen(
> - self.WorkspaceDir,
> - self.PlatformFile,
> - BuildTarget,
> - ToolChain,
> - self.ArchList,
> - self.BuildDatabase,
> - self.TargetTxt,
> - self.ToolDef,
> - self.Fdf,
> - self.FdList,
> - self.FvList,
> - self.CapList,
> - self.SkuId,
> - self.UniFlag
> - )
> FvDir = Wa.FvDir
> if not os.path.exists(FvDir):
> continue
>
> for Arch in self.ArchList:
> @@ -2285,11 +2373,10 @@ class Build():
> if not self.SpawnMode or self.Target not in ["", "all"]:
> self.SpawnMode = False
> self._BuildPlatform()
> else:
> self._MultiThreadBuildPlatform()
> - self.CreateGuidedSectionToolsFile()
> else:
> self.SpawnMode = False
> self._BuildModule()
>
> if self.Target == 'cleanall':
> --
> 2.20.1.windows.1
>
>
>
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen
[not found] ` <15C18FCC38905317.10639@groups.io>
@ 2019-09-05 14:05 ` Bob Feng
2019-09-09 6:58 ` Liming Gao
[not found] ` <15C2B2E9C55F0B32.16248@groups.io>
0 siblings, 2 replies; 6+ messages in thread
From: Bob Feng @ 2019-09-05 14:05 UTC (permalink / raw)
To: devel@edk2.groups.io, Feng, Bob C, Gao, Liming; +Cc: Shi, Steven
Sorry for the incomplete mail.
I verified this patch on Ovmf, Emulator, MinPlatforms and Intel Server Platform.
Thanks,
Bob
-----Original Message-----
From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Bob Feng
Sent: Thursday, September 5, 2019 10:04 PM
To: Gao, Liming <liming.gao@intel.com>; devel@edk2.groups.io
Cc: Shi, Steven <steven.shi@intel.com>
Subject: Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen
Liming,
Yes, this change only impact -u option.
I separate _MultiThreadBuildPlatform function into two functions and create 2 new functions specially for IgnoreAutoGen, so the there looks big change for build.py in this patch. I verified this patch on Ovmf, E
-Bob
-----Original Message-----
From: Gao, Liming
Sent: Thursday, September 5, 2019 9:12 PM
To: devel@edk2.groups.io; Feng, Bob C <bob.c.feng@intel.com>
Cc: Shi, Steven <steven.shi@intel.com>
Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen
Bob:
Does this change only impact -u option behavior?
Thanks
Liming
> -----Original Message-----
> From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of
> Bob Feng
> Sent: Thursday, September 5, 2019 5:05 PM
> To: devel@edk2.groups.io
> Cc: Gao, Liming <liming.gao@intel.com>; Shi, Steven
> <steven.shi@intel.com>; Feng, Bob C <bob.c.feng@intel.com>
> Subject: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for
> IgnoreAutoGen
>
> https://bugzilla.tianocore.org/show_bug.cgi?id=2080
>
> This patch is to improve build -u option to re-use
> GlobalVar_<platformguid>_<arch>.bin file which is introduced by
> multiple-process-autogen feature.
>
> Cc: Liming Gao <liming.gao@intel.com>
> Cc: Steven Shi <steven.shi@intel.com>
> Signed-off-by: Bob Feng <bob.c.feng@intel.com>
> ---
> .../Source/Python/AutoGen/AutoGenWorker.py | 2 +-
> .../Source/Python/AutoGen/ModuleAutoGen.py | 4 +-
> .../Python/AutoGen/ModuleAutoGenHelper.py | 29 ++
> .../Source/Python/AutoGen/PlatformAutoGen.py | 2 +
> .../Source/Python/AutoGen/WorkspaceAutoGen.py | 1 -
> BaseTools/Source/Python/build/build.py | 355 +++++++++++-------
> 6 files changed, 255 insertions(+), 138 deletions(-)
>
> diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> index 2e68538b1cb4..f488ae9d5f80 100755
> --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process):
> Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
> if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
> Ma.GenModuleFilesHash(GlobalData.gCacheIR)
> Ma.GenPreMakefileHash(GlobalData.gCacheIR)
> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
> - continue
> + continue
>
> Ma.CreateCodeFile(False)
>
> Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File,
> Ma.Arch),[]))
>
> if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> index 3bb7e91154ac..5f28681e3146 100755
> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen):
> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
> self.GenModuleFilesHash(gDict)
>
> if not (self.MetaFile.Path, self.Arch) in gDict or \
> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
> - EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
> self.Arch))
> - return
> + EdkLogger.quiet("[cache warning]: Cannot generate
> + ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
> self.Arch))
> + return
>
> # Initialze hash object
> m = hashlib.md5()
>
> # Add Platform level hash
> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> index c7591253debd..9dd93b9beb12 100644
> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> @@ -234,22 +234,51 @@ class AutoGenInfo(object):
> #
> #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in
> multiple thread # class WorkSpaceInfo(AutoGenInfo):
> def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch):
> + if not hasattr(self, "_Init"):
> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch)
> + self._Init = True
> + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch):
> self._SrcTimeStamp = 0
> self.Db = BuildDB
> self.BuildDatabase = self.Db.BuildObject
> self.Target = Target
> self.ToolChain = ToolChain
> self.WorkspaceDir = Workspace
> self.ActivePlatform = MetaFile
> self.ArchList = Arch
> + self.AutoGenObjectList = []
> + @property
> + def BuildDir(self):
> + return self.AutoGenObjectList[0].BuildDir
>
> + @property
> + def Name(self):
> + return self.AutoGenObjectList[0].Platform.PlatformName
> +
> + @property
> + def FlashDefinition(self):
> + return self.AutoGenObjectList[0].Platform.FlashDefinition
> + @property
> + def GenFdsCommandDict(self):
> + FdsCommandDict = self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict")
> + if FdsCommandDict:
> + return FdsCommandDict
> + return {}
> +
> + @cached_property
> + def FvDir(self):
> + return os.path.join(self.BuildDir, TAB_FV_DIRECTORY)
>
> class PlatformInfo(AutoGenInfo):
> def __init__(self, Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
> + if not hasattr(self, "_Init"):
> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe)
> + self._Init = True
> + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
> self.Wa = Workspace
> self.WorkspaceDir = self.Wa.WorkspaceDir
> self.MetaFile = MetaFile
> self.Arch = Arch
> self.Target = Target
> diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
> b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
> index 565424a95ead..1e17b6687129 100644
> --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
> +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
> @@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen):
> key = (Ma.MetaFile.File, self.Arch)
> if key in FfsCommand:
> Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key])
> else:
> Ma.CreateMakeFile(CreateModuleMakeFile)
> + self.CreateLibModuelDirs()
>
> + def CreateLibModuelDirs(self):
> # no need to create makefile for the platform more than once
> if self.IsMakeFileCreated:
> return
>
> # create library/module build dirs for platform diff --git
> a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
> b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
> index 24942674721f..f9d2c216169c 100644
> --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
> +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
> @@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen):
> self._CheckPcdDefineAndType()
>
> self.CreateBuildOptionsFile()
> self.CreatePcdTokenNumberFile()
> self.CreateModuleHashInfo()
> - GlobalData.gAutoGenPhase = False
>
> #
> # Merge Arch
> #
> def MergeArch(self):
> diff --git a/BaseTools/Source/Python/build/build.py
> b/BaseTools/Source/Python/build/build.py
> index 0406ac314b65..db57c3e526da 100755
> --- a/BaseTools/Source/Python/build/build.py
> +++ b/BaseTools/Source/Python/build/build.py
> @@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import
> PatchBinaryFile
>
> import Common.GlobalData as GlobalData from GenFds.GenFds import
> GenFds, GenFdsApi import multiprocessing as mp from multiprocessing
> import Manager
> +from AutoGen.DataPipe import MemoryDataPipe from
> +AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo from
> +GenFds.FdfParser import FdfParser
>
>
> ## standard targets of build command
> gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries',
> 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
>
> @@ -837,11 +840,11 @@ class Build():
> for PcdMa in PcdMaList:
> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
> PcdMa.GenModuleFilesHash(share_data)
> PcdMa.GenPreMakefileHash(share_data)
> if PcdMa.CanSkipbyPreMakefileCache(share_data):
> - continue
> + continue
>
> PcdMa.CreateCodeFile(False)
> PcdMa.CreateMakeFile(False,GenFfsList =
> DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File,
> PcdMa.Arch),[]))
>
> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
> @@ -1218,18 +1221,28 @@ class Build():
> mqueue = mp.Queue()
> for m in AutoGenObject.GetAllModuleInfo:
> mqueue.put(m)
>
> AutoGenObject.DataPipe.DataContainer = {"CommandTarget":
> self.Target}
> + AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}
> + AutoGenObject.CreateLibModuelDirs()
> + AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
> + AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
> + AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict":
> + AutoGenObject.Workspace.GenFdsCommandDict}
> self.Progress.Start("Generating makefile and code")
> data_pipe_file = os.path.join(AutoGenObject.BuildDir,
> "GlobalVar_%s_%s.bin" %
> (str(AutoGenObject.Guid),AutoGenObject.Arch))
> AutoGenObject.DataPipe.dump(data_pipe_file)
> autogen_rt,errorcode = self.StartAutoGen(mqueue,
> AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList,
> GlobalData.gCacheIR)
> + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
> + with open(AutoGenIdFile,"w") as fw:
> + fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList)))
> + fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir)
> + fw.write("PlatformGuid=%s\n" %
> + str(AutoGenObject.Guid))
> self.Progress.Stop("done!")
> if not autogen_rt:
> self.AutoGenMgr.TerminateWorkers()
> - self.AutoGenMgr.join(0.1)
> + self.AutoGenMgr.join(1)
> raise FatalError(errorcode)
> AutoGenObject.CreateCodeFile(False)
> AutoGenObject.CreateMakeFile(False)
> else:
> # always recreate top/platform makefile when clean, just
> in case of inconsistency @@ -1778,10 +1791,11 @@ class Build():
> self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
> #
> # Save MAP buffer into MAP file.
> #
> self._SaveMapFile (MapBuffer, Wa)
> + self.CreateGuidedSectionToolsFile(Wa)
>
> ## Build active module for different build targets, different tool chains and different archs
> #
> def _BuildModule(self):
> for BuildTarget in self.BuildTargetList:
> @@ -1852,13 +1866,13 @@ class Build():
>
> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
> Ma.GenModuleFilesHash(GlobalData.gCacheIR)
> Ma.GenPreMakefileHash(GlobalData.gCacheIR)
> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
> - self.HashSkipModules.append(Ma)
> - EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
> - continue
> + self.HashSkipModules.append(Ma)
> + EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
> + continue
>
> # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
> if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
> # for target which must generate AutoGen code and makefile
> if not self.SkipAutoGen or self.Target == 'genc':
> @@ -1987,125 +2001,221 @@ class Build():
> GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData)
> for Cmd in GenFfsDict:
> tmpInf, tmpArch = GenFfsDict[Cmd]
> CmdSetDict[tmpInf, tmpArch].add(Cmd)
> return CmdSetDict
> + def VerifyAutoGenFiles(self):
> + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
> + try:
> + with open(AutoGenIdFile) as fd:
> + lines = fd.readlines()
> + except:
> + return None
> + for line in lines:
> + if "Arch" in line:
> + ArchList = line.strip().split("=")[1].split("|")
> + if "BuildDir" in line:
> + BuildDir = line.split("=")[1].strip()
> + if "PlatformGuid" in line:
> + PlatformGuid = line.split("=")[1].strip()
> + GlobalVarList = []
> + for arch in ArchList:
> + global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch))
> + if not os.path.exists(global_var):
> + return None
> + GlobalVarList.append(global_var)
> + for global_var in GlobalVarList:
> + data_pipe = MemoryDataPipe()
> + data_pipe.load(global_var)
> + target = data_pipe.Get("P_Info").get("Target")
> + toolchain = data_pipe.Get("P_Info").get("ToolChain")
> + archlist = data_pipe.Get("P_Info").get("ArchList")
> + Arch = data_pipe.Get("P_Info").get("Arch")
> + active_p = data_pipe.Get("P_Info").get("ActivePlatform")
> + workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir")
> + PackagesPath = os.getenv("PACKAGES_PATH")
> + mws.setWs(workspacedir, PackagesPath)
> + LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList")
> + ModuleBuildDirectoryList =
> + data_pipe.Get("ModuleBuildDirectoryList")
> +
> + for m_build_dir in LibraryBuildDirectoryList:
> + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
> + return None
> + for m_build_dir in ModuleBuildDirectoryList:
> + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
> + return None
> + Wa = WorkSpaceInfo(
> + workspacedir,active_p,target,toolchain,archlist
> + )
> + Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe)
> + Wa.AutoGenObjectList.append(Pa)
> + return Wa
> + def SetupMakeSetting(self,Wa):
> + BuildModules = []
> + for Pa in Wa.AutoGenObjectList:
> + for m in Pa._MbList:
> + ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe)
> + BuildModules.append(ma)
> + fdf_file = Wa.FlashDefinition
> + if fdf_file:
> + Fdf = FdfParser(fdf_file.Path)
> + Fdf.ParseFile()
> + GlobalData.gFdfParser = Fdf
> + if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:
> + FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]
> + for FdRegion in FdDict.RegionList:
> + if str(FdRegion.RegionType) is 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):
> + if int(FdRegion.Offset) % 8 != 0:
> + EdkLogger.error("build", FORMAT_INVALID,
> + 'The VPD Base Address %s must be 8-byte aligned.' %
> (FdRegion.Offset))
> + Wa.FdfProfile = Fdf.Profile
> + self.Fdf = Fdf
> + else:
> + self.Fdf = None
> + return BuildModules
>
> ## Build a platform in multi-thread mode
> #
> - def _MultiThreadBuildPlatform(self):
> - SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
> - for BuildTarget in self.BuildTargetList:
> - GlobalData.gGlobalDefines['TARGET'] = BuildTarget
> - index = 0
> - for ToolChain in self.ToolChainList:
> - WorkspaceAutoGenTime = time.time()
> - GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
> - GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
> - GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
> - index += 1
> - Wa = WorkspaceAutoGen(
> - self.WorkspaceDir,
> - self.PlatformFile,
> - BuildTarget,
> - ToolChain,
> - self.ArchList,
> - self.BuildDatabase,
> - self.TargetTxt,
> - self.ToolDef,
> - self.Fdf,
> - self.FdList,
> - self.FvList,
> - self.CapList,
> - self.SkuId,
> - self.UniFlag,
> - self.Progress
> - )
> - self.Fdf = Wa.FdfFile
> - self.LoadFixAddress = Wa.Platform.LoadFixAddress
> - self.BuildReport.AddPlatformReport(Wa)
> - Wa.CreateMakeFile(False)
> + def PerformAutoGen(self,BuildTarget,ToolChain):
> + WorkspaceAutoGenTime = time.time()
> + Wa = WorkspaceAutoGen(
> + self.WorkspaceDir,
> + self.PlatformFile,
> + BuildTarget,
> + ToolChain,
> + self.ArchList,
> + self.BuildDatabase,
> + self.TargetTxt,
> + self.ToolDef,
> + self.Fdf,
> + self.FdList,
> + self.FvList,
> + self.CapList,
> + self.SkuId,
> + self.UniFlag,
> + self.Progress
> + )
> + self.Fdf = Wa.FdfFile
> + self.LoadFixAddress = Wa.Platform.LoadFixAddress
> + self.BuildReport.AddPlatformReport(Wa)
> + Wa.CreateMakeFile(False)
>
> # Add ffs build to makefile
> - CmdListDict = {}
> - if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
> - CmdListDict = self._GenFfsCmd(Wa.ArchList)
> + CmdListDict = {}
> + if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
> + CmdListDict = self._GenFfsCmd(Wa.ArchList)
>
> - # Add Platform and Package level hash in share_data for module hash calculation later
> - if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
> - GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
> - for PkgName in GlobalData.gPackageHash.keys():
> - GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]
> + # Add Platform and Package level hash in share_data for module hash calculation later
> + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
> + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
> + for PkgName in GlobalData.gPackageHash.keys():
> + GlobalData.gCacheIR[(PkgName, 'PackageHash')] =
> + GlobalData.gPackageHash[PkgName]
>
> - # multi-thread exit flag
> - ExitFlag = threading.Event()
> - ExitFlag.clear()
> - self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
> - self.BuildModules = []
> - TotalModules = []
> - for Arch in Wa.ArchList:
> - PcdMaList = []
> - AutoGenStart = time.time()
> - GlobalData.gGlobalDefines['ARCH'] = Arch
> - Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
> - if Pa is None:
> - continue
> - ModuleList = []
> - for Inf in Pa.Platform.Modules:
> - ModuleList.append(Inf)
> + self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
> + BuildModules = []
> + TotalModules = []
> + for Arch in Wa.ArchList:
> + PcdMaList = []
> + AutoGenStart = time.time()
> + GlobalData.gGlobalDefines['ARCH'] = Arch
> + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
> + if Pa is None:
> + continue
> + ModuleList = []
> + for Inf in Pa.Platform.Modules:
> + ModuleList.append(Inf)
> # Add the INF only list in FDF
> - if GlobalData.gFdfParser is not None:
> - for InfName in GlobalData.gFdfParser.Profile.InfList:
> - Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
> - if Inf in Pa.Platform.Modules:
> - continue
> - ModuleList.append(Inf)
> - Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
> - Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
> - Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
> - for Module in ModuleList:
> + if GlobalData.gFdfParser is not None:
> + for InfName in GlobalData.gFdfParser.Profile.InfList:
> + Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
> + if Inf in Pa.Platform.Modules:
> + continue
> + ModuleList.append(Inf)
> + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
> + Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
> + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
> + Pa.CreateLibModuelDirs()
> + Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}
> + Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}
> + Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict}
> + ModuleCodaFile = {}
> + for ma in Pa.ModuleAutoGenList:
> +
> + ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFil
> + e.Path)] = [item.Target for item in
> ma.CodaTargetList]
> + Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}
> + for Module in ModuleList:
> # Get ModuleAutoGen object to generate C code file and makefile
> - Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
> + Ma = ModuleAutoGen(Wa, Module, BuildTarget,
> + ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
>
> - if Ma is None:
> + if Ma is None:
> + continue
> + if Ma.PcdIsDriver:
> + Ma.PlatformInfo = Pa
> + Ma.Workspace = Wa
> + PcdMaList.append(Ma)
> + TotalModules.append(Ma)
> + # Initialize all modules in tracking to 'FAIL'
> + GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
> +
> +
> + mqueue = mp.Queue()
> + for m in Pa.GetAllModuleInfo:
> + mqueue.put(m)
> + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
> + Pa.DataPipe.dump(data_pipe_file)
> +
> + autogen_rt, errorcode = self.StartAutoGen(mqueue,
> + Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR)
> +
> + # Skip cache hit modules
> + if GlobalData.gBinCacheSource:
> + for Ma in TotalModules:
> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
> + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:
> + self.HashSkipModules.append(Ma)
> + continue
> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
> + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:
> + self.HashSkipModules.append(Ma)
> continue
> - if Ma.PcdIsDriver:
> - Ma.PlatformInfo = Pa
> - Ma.Workspace = Wa
> - PcdMaList.append(Ma)
> - TotalModules.append(Ma)
> - # Initialize all modules in tracking to 'FAIL'
> - GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
> + BuildModules.append(Ma)
> + else:
> + BuildModules.extend(TotalModules)
>
> - mqueue = mp.Queue()
> - for m in Pa.GetAllModuleInfo:
> - mqueue.put(m)
> - data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
> - Pa.DataPipe.dump(data_pipe_file)
> - autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,
> GlobalData.gCacheIR)
> + if not autogen_rt:
> + self.AutoGenMgr.TerminateWorkers()
> + self.AutoGenMgr.join(1)
> + raise FatalError(errorcode)
> + self.AutoGenTime += int(round((time.time() - AutoGenStart)))
> + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
> + with open(AutoGenIdFile,"w") as fw:
> + fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))
> + fw.write("BuildDir=%s\n" % Wa.BuildDir)
> + fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid))
> + self.Progress.Stop("done!")
> + return Wa, BuildModules
>
> - # Skip cache hit modules
> - if GlobalData.gBinCacheSource:
> - for Ma in TotalModules:
> - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
> - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:
> - self.HashSkipModules.append(Ma)
> - continue
> - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
> - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:
> - self.HashSkipModules.append(Ma)
> - continue
> - self.BuildModules.append(Ma)
> + def _MultiThreadBuildPlatform(self):
> + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
> + for BuildTarget in self.BuildTargetList:
> + GlobalData.gGlobalDefines['TARGET'] = BuildTarget
> + index = 0
> + for ToolChain in self.ToolChainList:
> + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
> + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
> + GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
> + index += 1
> + ExitFlag = threading.Event()
> + ExitFlag.clear()
> + if self.SkipAutoGen:
> + Wa = self.VerifyAutoGenFiles()
> + if Wa is None:
> + Wa, self.BuildModules =
> + self.PerformAutoGen(BuildTarget,ToolChain)
> else:
> - self.BuildModules.extend(TotalModules)
> -
> - if not autogen_rt:
> - self.AutoGenMgr.TerminateWorkers()
> - self.AutoGenMgr.join(0.1)
> - raise FatalError(errorcode)
> - self.AutoGenTime += int(round((time.time() - AutoGenStart)))
> - self.Progress.Stop("done!")
> + GlobalData.gAutoGenPhase = True
> + self.BuildModules = self.SetupMakeSetting(Wa)
> + else:
> + Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
> + Pa = Wa.AutoGenObjectList[0]
> + GlobalData.gAutoGenPhase = False
>
> if GlobalData.gBinCacheSource:
> EdkLogger.quiet("Total cache hit driver num: %s,
> cache miss driver num: %s" % (len(set(self.HashSkipModules)),
> len(set(self.BuildModules))))
> CacheHitMa = set()
> CacheNotHitMa = set() @@ -2136,18 +2246,17 @@
> class Build():
> # Start task scheduler
> if not BuildTask.IsOnGoing():
>
> BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
>
> # in case there's an interruption. we need a full version of makefile for platform
> - Pa.CreateMakeFile(False)
> +
> if BuildTask.HasError():
> self.invalidateHash()
> EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
> self.MakeTime += int(round((time.time() -
> MakeStart)))
>
> MakeContiue = time.time()
> -
> #
> #
> # All modules have been put in build tasks queue. Tell task scheduler
> # to exit if all tasks are completed
> #
> @@ -2175,17 +2284,12 @@ class Build():
> if (Arch == 'IA32' or Arch == 'ARM') and
> self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
> EdkLogger.error("build",
> PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
> #
> # Get Module List
> #
> - ModuleList = {}
> - for Pa in Wa.AutoGenObjectList:
> - for Ma in Pa.ModuleAutoGenList:
> - if Ma is None:
> - continue
> - if not Ma.IsLibrary:
> - ModuleList[Ma.Guid.upper()] = Ma
> + ModuleList = {ma.Guid.upper():ma for ma in
> + self.BuildModules}
> +
> #
> # Rebase module to the preferred memory address before GenFds
> #
> MapBuffer = []
> if self.LoadFixAddress != 0:
> @@ -2206,33 +2310,17 @@ class Build():
> self.GenFdsTime += int(round((time.time() - GenFdsStart)))
> #
> # Save MAP buffer into MAP file.
> #
> self._SaveMapFile(MapBuffer, Wa)
> + self.CreateGuidedSectionToolsFile(Wa)
> self.invalidateHash()
> -
> ## Generate GuidedSectionTools.txt in the FV directories.
> #
> - def CreateGuidedSectionToolsFile(self):
> + def CreateGuidedSectionToolsFile(self,Wa):
> for BuildTarget in self.BuildTargetList:
> for ToolChain in self.ToolChainList:
> - Wa = WorkspaceAutoGen(
> - self.WorkspaceDir,
> - self.PlatformFile,
> - BuildTarget,
> - ToolChain,
> - self.ArchList,
> - self.BuildDatabase,
> - self.TargetTxt,
> - self.ToolDef,
> - self.Fdf,
> - self.FdList,
> - self.FvList,
> - self.CapList,
> - self.SkuId,
> - self.UniFlag
> - )
> FvDir = Wa.FvDir
> if not os.path.exists(FvDir):
> continue
>
> for Arch in self.ArchList:
> @@ -2285,11 +2373,10 @@ class Build():
> if not self.SpawnMode or self.Target not in ["", "all"]:
> self.SpawnMode = False
> self._BuildPlatform()
> else:
> self._MultiThreadBuildPlatform()
> - self.CreateGuidedSectionToolsFile()
> else:
> self.SpawnMode = False
> self._BuildModule()
>
> if self.Target == 'cleanall':
> --
> 2.20.1.windows.1
>
>
>
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen
2019-09-05 14:05 ` Bob Feng
@ 2019-09-09 6:58 ` Liming Gao
[not found] ` <15C2B2E9C55F0B32.16248@groups.io>
1 sibling, 0 replies; 6+ messages in thread
From: Liming Gao @ 2019-09-09 6:58 UTC (permalink / raw)
To: Feng, Bob C, devel@edk2.groups.io; +Cc: Shi, Steven
Liming Gao <liming.gao@intel.com>
>-----Original Message-----
>From: Feng, Bob C
>Sent: Thursday, September 05, 2019 10:06 PM
>To: devel@edk2.groups.io; Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming
><liming.gao@intel.com>
>Cc: Shi, Steven <steven.shi@intel.com>
>Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for
>IgnoreAutoGen
>
>Sorry for the incomplete mail.
>
>I verified this patch on Ovmf, Emulator, MinPlatforms and Intel Server
>Platform.
>
>Thanks,
>Bob
>
>-----Original Message-----
>From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Bob Feng
>Sent: Thursday, September 5, 2019 10:04 PM
>To: Gao, Liming <liming.gao@intel.com>; devel@edk2.groups.io
>Cc: Shi, Steven <steven.shi@intel.com>
>Subject: Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for
>IgnoreAutoGen
>
>Liming,
>
>Yes, this change only impact -u option.
>
>I separate _MultiThreadBuildPlatform function into two functions and create 2
>new functions specially for IgnoreAutoGen, so the there looks big change for
>build.py in this patch. I verified this patch on Ovmf, E
>
>-Bob
>
>-----Original Message-----
>From: Gao, Liming
>Sent: Thursday, September 5, 2019 9:12 PM
>To: devel@edk2.groups.io; Feng, Bob C <bob.c.feng@intel.com>
>Cc: Shi, Steven <steven.shi@intel.com>
>Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for
>IgnoreAutoGen
>
>Bob:
> Does this change only impact -u option behavior?
>
>Thanks
>Liming
>> -----Original Message-----
>> From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of
>> Bob Feng
>> Sent: Thursday, September 5, 2019 5:05 PM
>> To: devel@edk2.groups.io
>> Cc: Gao, Liming <liming.gao@intel.com>; Shi, Steven
>> <steven.shi@intel.com>; Feng, Bob C <bob.c.feng@intel.com>
>> Subject: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for
>> IgnoreAutoGen
>>
>> https://bugzilla.tianocore.org/show_bug.cgi?id=2080
>>
>> This patch is to improve build -u option to re-use
>> GlobalVar_<platformguid>_<arch>.bin file which is introduced by
>> multiple-process-autogen feature.
>>
>> Cc: Liming Gao <liming.gao@intel.com>
>> Cc: Steven Shi <steven.shi@intel.com>
>> Signed-off-by: Bob Feng <bob.c.feng@intel.com>
>> ---
>> .../Source/Python/AutoGen/AutoGenWorker.py | 2 +-
>> .../Source/Python/AutoGen/ModuleAutoGen.py | 4 +-
>> .../Python/AutoGen/ModuleAutoGenHelper.py | 29 ++
>> .../Source/Python/AutoGen/PlatformAutoGen.py | 2 +
>> .../Source/Python/AutoGen/WorkspaceAutoGen.py | 1 -
>> BaseTools/Source/Python/build/build.py | 355 +++++++++++-------
>> 6 files changed, 255 insertions(+), 138 deletions(-)
>>
>> diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
>> b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
>> index 2e68538b1cb4..f488ae9d5f80 100755
>> --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
>> +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
>> @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process):
>> Ma.ReferenceModules =
>Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
>> if GlobalData.gBinCacheSource and CommandTarget in [None, "",
>"all"]:
>> Ma.GenModuleFilesHash(GlobalData.gCacheIR)
>> Ma.GenPreMakefileHash(GlobalData.gCacheIR)
>> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
>> - continue
>> + continue
>>
>> Ma.CreateCodeFile(False)
>>
>> Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File,
>> Ma.Arch),[]))
>>
>> if GlobalData.gBinCacheSource and CommandTarget in [None, "",
>"all"]:
>> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
>> b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
>> index 3bb7e91154ac..5f28681e3146 100755
>> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
>> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
>> @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen):
>> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
>> self.GenModuleFilesHash(gDict)
>>
>> if not (self.MetaFile.Path, self.Arch) in gDict or \
>> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
>> - EdkLogger.quiet("[cache warning]: Cannot generate
>ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
>> self.Arch))
>> - return
>> + EdkLogger.quiet("[cache warning]: Cannot generate
>> + ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
>> self.Arch))
>> + return
>>
>> # Initialze hash object
>> m = hashlib.md5()
>>
>> # Add Platform level hash
>> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
>> b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
>> index c7591253debd..9dd93b9beb12 100644
>> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
>> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
>> @@ -234,22 +234,51 @@ class AutoGenInfo(object):
>> #
>> #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in
>> multiple thread # class WorkSpaceInfo(AutoGenInfo):
>> def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch):
>> + if not hasattr(self, "_Init"):
>> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch)
>> + self._Init = True
>> + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch):
>> self._SrcTimeStamp = 0
>> self.Db = BuildDB
>> self.BuildDatabase = self.Db.BuildObject
>> self.Target = Target
>> self.ToolChain = ToolChain
>> self.WorkspaceDir = Workspace
>> self.ActivePlatform = MetaFile
>> self.ArchList = Arch
>> + self.AutoGenObjectList = []
>> + @property
>> + def BuildDir(self):
>> + return self.AutoGenObjectList[0].BuildDir
>>
>> + @property
>> + def Name(self):
>> + return self.AutoGenObjectList[0].Platform.PlatformName
>> +
>> + @property
>> + def FlashDefinition(self):
>> + return self.AutoGenObjectList[0].Platform.FlashDefinition
>> + @property
>> + def GenFdsCommandDict(self):
>> + FdsCommandDict =
>self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict")
>> + if FdsCommandDict:
>> + return FdsCommandDict
>> + return {}
>> +
>> + @cached_property
>> + def FvDir(self):
>> + return os.path.join(self.BuildDir, TAB_FV_DIRECTORY)
>>
>> class PlatformInfo(AutoGenInfo):
>> def __init__(self, Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
>> + if not hasattr(self, "_Init"):
>> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe)
>> + self._Init = True
>> + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
>> self.Wa = Workspace
>> self.WorkspaceDir = self.Wa.WorkspaceDir
>> self.MetaFile = MetaFile
>> self.Arch = Arch
>> self.Target = Target
>> diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
>> b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
>> index 565424a95ead..1e17b6687129 100644
>> --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
>> +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
>> @@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen):
>> key = (Ma.MetaFile.File, self.Arch)
>> if key in FfsCommand:
>> Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key])
>> else:
>> Ma.CreateMakeFile(CreateModuleMakeFile)
>> + self.CreateLibModuelDirs()
>>
>> + def CreateLibModuelDirs(self):
>> # no need to create makefile for the platform more than once
>> if self.IsMakeFileCreated:
>> return
>>
>> # create library/module build dirs for platform diff --git
>> a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
>> b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
>> index 24942674721f..f9d2c216169c 100644
>> --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
>> +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
>> @@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen):
>> self._CheckPcdDefineAndType()
>>
>> self.CreateBuildOptionsFile()
>> self.CreatePcdTokenNumberFile()
>> self.CreateModuleHashInfo()
>> - GlobalData.gAutoGenPhase = False
>>
>> #
>> # Merge Arch
>> #
>> def MergeArch(self):
>> diff --git a/BaseTools/Source/Python/build/build.py
>> b/BaseTools/Source/Python/build/build.py
>> index 0406ac314b65..db57c3e526da 100755
>> --- a/BaseTools/Source/Python/build/build.py
>> +++ b/BaseTools/Source/Python/build/build.py
>> @@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import
>> PatchBinaryFile
>>
>> import Common.GlobalData as GlobalData from GenFds.GenFds import
>> GenFds, GenFdsApi import multiprocessing as mp from multiprocessing
>> import Manager
>> +from AutoGen.DataPipe import MemoryDataPipe from
>> +AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo
>from
>> +GenFds.FdfParser import FdfParser
>>
>>
>> ## standard targets of build command
>> gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries',
>> 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
>>
>> @@ -837,11 +840,11 @@ class Build():
>> for PcdMa in PcdMaList:
>> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
>> PcdMa.GenModuleFilesHash(share_data)
>> PcdMa.GenPreMakefileHash(share_data)
>> if PcdMa.CanSkipbyPreMakefileCache(share_data):
>> - continue
>> + continue
>>
>> PcdMa.CreateCodeFile(False)
>> PcdMa.CreateMakeFile(False,GenFfsList =
>> DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File,
>> PcdMa.Arch),[]))
>>
>> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
>> @@ -1218,18 +1221,28 @@ class Build():
>> mqueue = mp.Queue()
>> for m in AutoGenObject.GetAllModuleInfo:
>> mqueue.put(m)
>>
>> AutoGenObject.DataPipe.DataContainer = {"CommandTarget":
>> self.Target}
>> + AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp":
>AutoGenObject.Workspace._SrcTimeStamp}
>> + AutoGenObject.CreateLibModuelDirs()
>> + AutoGenObject.DataPipe.DataContainer =
>{"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
>> + AutoGenObject.DataPipe.DataContainer =
>{"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
>> + AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict":
>> + AutoGenObject.Workspace.GenFdsCommandDict}
>> self.Progress.Start("Generating makefile and code")
>> data_pipe_file = os.path.join(AutoGenObject.BuildDir,
>> "GlobalVar_%s_%s.bin" %
>> (str(AutoGenObject.Guid),AutoGenObject.Arch))
>> AutoGenObject.DataPipe.dump(data_pipe_file)
>> autogen_rt,errorcode = self.StartAutoGen(mqueue,
>> AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList,
>> GlobalData.gCacheIR)
>> + AutoGenIdFile =
>os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
>> + with open(AutoGenIdFile,"w") as fw:
>> + fw.write("Arch=%s\n" %
>"|".join((AutoGenObject.Workspace.ArchList)))
>> + fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir)
>> + fw.write("PlatformGuid=%s\n" %
>> + str(AutoGenObject.Guid))
>> self.Progress.Stop("done!")
>> if not autogen_rt:
>> self.AutoGenMgr.TerminateWorkers()
>> - self.AutoGenMgr.join(0.1)
>> + self.AutoGenMgr.join(1)
>> raise FatalError(errorcode)
>> AutoGenObject.CreateCodeFile(False)
>> AutoGenObject.CreateMakeFile(False)
>> else:
>> # always recreate top/platform makefile when clean, just
>> in case of inconsistency @@ -1778,10 +1791,11 @@ class Build():
>> self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
>> #
>> # Save MAP buffer into MAP file.
>> #
>> self._SaveMapFile (MapBuffer, Wa)
>> + self.CreateGuidedSectionToolsFile(Wa)
>>
>> ## Build active module for different build targets, different tool chains
>and different archs
>> #
>> def _BuildModule(self):
>> for BuildTarget in self.BuildTargetList:
>> @@ -1852,13 +1866,13 @@ class Build():
>>
>> if GlobalData.gBinCacheSource and self.Target in [None, "",
>"all"]:
>> Ma.GenModuleFilesHash(GlobalData.gCacheIR)
>> Ma.GenPreMakefileHash(GlobalData.gCacheIR)
>> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
>> - self.HashSkipModules.append(Ma)
>> - EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path,
>Ma.Arch))
>> - continue
>> + self.HashSkipModules.append(Ma)
>> + EdkLogger.quiet("cache hit: %s[%s]" %
>(Ma.MetaFile.Path, Ma.Arch))
>> + continue
>>
>> # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run',
>'fds'
>> if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
>> # for target which must generate AutoGen code and
>makefile
>> if not self.SkipAutoGen or self.Target == 'genc':
>> @@ -1987,125 +2001,221 @@ class Build():
>> GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self,
>ArchList, GlobalData)
>> for Cmd in GenFfsDict:
>> tmpInf, tmpArch = GenFfsDict[Cmd]
>> CmdSetDict[tmpInf, tmpArch].add(Cmd)
>> return CmdSetDict
>> + def VerifyAutoGenFiles(self):
>> + AutoGenIdFile =
>os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
>> + try:
>> + with open(AutoGenIdFile) as fd:
>> + lines = fd.readlines()
>> + except:
>> + return None
>> + for line in lines:
>> + if "Arch" in line:
>> + ArchList = line.strip().split("=")[1].split("|")
>> + if "BuildDir" in line:
>> + BuildDir = line.split("=")[1].strip()
>> + if "PlatformGuid" in line:
>> + PlatformGuid = line.split("=")[1].strip()
>> + GlobalVarList = []
>> + for arch in ArchList:
>> + global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" %
>(str(PlatformGuid),arch))
>> + if not os.path.exists(global_var):
>> + return None
>> + GlobalVarList.append(global_var)
>> + for global_var in GlobalVarList:
>> + data_pipe = MemoryDataPipe()
>> + data_pipe.load(global_var)
>> + target = data_pipe.Get("P_Info").get("Target")
>> + toolchain = data_pipe.Get("P_Info").get("ToolChain")
>> + archlist = data_pipe.Get("P_Info").get("ArchList")
>> + Arch = data_pipe.Get("P_Info").get("Arch")
>> + active_p = data_pipe.Get("P_Info").get("ActivePlatform")
>> + workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir")
>> + PackagesPath = os.getenv("PACKAGES_PATH")
>> + mws.setWs(workspacedir, PackagesPath)
>> + LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList")
>> + ModuleBuildDirectoryList =
>> + data_pipe.Get("ModuleBuildDirectoryList")
>> +
>> + for m_build_dir in LibraryBuildDirectoryList:
>> + if not
>os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[Ge
>nMake.gMakeType])):
>> + return None
>> + for m_build_dir in ModuleBuildDirectoryList:
>> + if not
>os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[Ge
>nMake.gMakeType])):
>> + return None
>> + Wa = WorkSpaceInfo(
>> + workspacedir,active_p,target,toolchain,archlist
>> + )
>> + Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe)
>> + Wa.AutoGenObjectList.append(Pa)
>> + return Wa
>> + def SetupMakeSetting(self,Wa):
>> + BuildModules = []
>> + for Pa in Wa.AutoGenObjectList:
>> + for m in Pa._MbList:
>> + ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget,
>Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe)
>> + BuildModules.append(ma)
>> + fdf_file = Wa.FlashDefinition
>> + if fdf_file:
>> + Fdf = FdfParser(fdf_file.Path)
>> + Fdf.ParseFile()
>> + GlobalData.gFdfParser = Fdf
>> + if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:
>> + FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]
>> + for FdRegion in FdDict.RegionList:
>> + if str(FdRegion.RegionType) is 'FILE' and
>self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):
>> + if int(FdRegion.Offset) % 8 != 0:
>> + EdkLogger.error("build", FORMAT_INVALID,
>> + 'The VPD Base Address %s must be 8-byte aligned.' %
>> (FdRegion.Offset))
>> + Wa.FdfProfile = Fdf.Profile
>> + self.Fdf = Fdf
>> + else:
>> + self.Fdf = None
>> + return BuildModules
>>
>> ## Build a platform in multi-thread mode
>> #
>> - def _MultiThreadBuildPlatform(self):
>> - SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE
>auto-generated\n', False)
>> - for BuildTarget in self.BuildTargetList:
>> - GlobalData.gGlobalDefines['TARGET'] = BuildTarget
>> - index = 0
>> - for ToolChain in self.ToolChainList:
>> - WorkspaceAutoGenTime = time.time()
>> - GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
>> - GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
>> - GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
>> - index += 1
>> - Wa = WorkspaceAutoGen(
>> - self.WorkspaceDir,
>> - self.PlatformFile,
>> - BuildTarget,
>> - ToolChain,
>> - self.ArchList,
>> - self.BuildDatabase,
>> - self.TargetTxt,
>> - self.ToolDef,
>> - self.Fdf,
>> - self.FdList,
>> - self.FvList,
>> - self.CapList,
>> - self.SkuId,
>> - self.UniFlag,
>> - self.Progress
>> - )
>> - self.Fdf = Wa.FdfFile
>> - self.LoadFixAddress = Wa.Platform.LoadFixAddress
>> - self.BuildReport.AddPlatformReport(Wa)
>> - Wa.CreateMakeFile(False)
>> + def PerformAutoGen(self,BuildTarget,ToolChain):
>> + WorkspaceAutoGenTime = time.time()
>> + Wa = WorkspaceAutoGen(
>> + self.WorkspaceDir,
>> + self.PlatformFile,
>> + BuildTarget,
>> + ToolChain,
>> + self.ArchList,
>> + self.BuildDatabase,
>> + self.TargetTxt,
>> + self.ToolDef,
>> + self.Fdf,
>> + self.FdList,
>> + self.FvList,
>> + self.CapList,
>> + self.SkuId,
>> + self.UniFlag,
>> + self.Progress
>> + )
>> + self.Fdf = Wa.FdfFile
>> + self.LoadFixAddress = Wa.Platform.LoadFixAddress
>> + self.BuildReport.AddPlatformReport(Wa)
>> + Wa.CreateMakeFile(False)
>>
>> # Add ffs build to makefile
>> - CmdListDict = {}
>> - if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
>> - CmdListDict = self._GenFfsCmd(Wa.ArchList)
>> + CmdListDict = {}
>> + if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
>> + CmdListDict = self._GenFfsCmd(Wa.ArchList)
>>
>> - # Add Platform and Package level hash in share_data for module
>hash calculation later
>> - if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
>> - GlobalData.gCacheIR[('PlatformHash')] =
>GlobalData.gPlatformHash
>> - for PkgName in GlobalData.gPackageHash.keys():
>> - GlobalData.gCacheIR[(PkgName, 'PackageHash')] =
>GlobalData.gPackageHash[PkgName]
>> + # Add Platform and Package level hash in share_data for module hash
>calculation later
>> + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
>> + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
>> + for PkgName in GlobalData.gPackageHash.keys():
>> + GlobalData.gCacheIR[(PkgName, 'PackageHash')] =
>> + GlobalData.gPackageHash[PkgName]
>>
>> - # multi-thread exit flag
>> - ExitFlag = threading.Event()
>> - ExitFlag.clear()
>> - self.AutoGenTime += int(round((time.time() -
>WorkspaceAutoGenTime)))
>> - self.BuildModules = []
>> - TotalModules = []
>> - for Arch in Wa.ArchList:
>> - PcdMaList = []
>> - AutoGenStart = time.time()
>> - GlobalData.gGlobalDefines['ARCH'] = Arch
>> - Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget,
>ToolChain, Arch)
>> - if Pa is None:
>> - continue
>> - ModuleList = []
>> - for Inf in Pa.Platform.Modules:
>> - ModuleList.append(Inf)
>> + self.AutoGenTime += int(round((time.time() -
>WorkspaceAutoGenTime)))
>> + BuildModules = []
>> + TotalModules = []
>> + for Arch in Wa.ArchList:
>> + PcdMaList = []
>> + AutoGenStart = time.time()
>> + GlobalData.gGlobalDefines['ARCH'] = Arch
>> + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain,
>Arch)
>> + if Pa is None:
>> + continue
>> + ModuleList = []
>> + for Inf in Pa.Platform.Modules:
>> + ModuleList.append(Inf)
>> # Add the INF only list in FDF
>> - if GlobalData.gFdfParser is not None:
>> - for InfName in GlobalData.gFdfParser.Profile.InfList:
>> - Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
>> - if Inf in Pa.Platform.Modules:
>> - continue
>> - ModuleList.append(Inf)
>> - Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
>> - Pa.DataPipe.DataContainer = {"Workspace_timestamp":
>Wa._SrcTimeStamp}
>> - Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
>> - for Module in ModuleList:
>> + if GlobalData.gFdfParser is not None:
>> + for InfName in GlobalData.gFdfParser.Profile.InfList:
>> + Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
>> + if Inf in Pa.Platform.Modules:
>> + continue
>> + ModuleList.append(Inf)
>> + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
>> + Pa.DataPipe.DataContainer = {"Workspace_timestamp":
>Wa._SrcTimeStamp}
>> + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
>> + Pa.CreateLibModuelDirs()
>> + Pa.DataPipe.DataContainer =
>{"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}
>> + Pa.DataPipe.DataContainer =
>{"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}
>> + Pa.DataPipe.DataContainer = {"FdsCommandDict":
>Wa.GenFdsCommandDict}
>> + ModuleCodaFile = {}
>> + for ma in Pa.ModuleAutoGenList:
>> +
>> + ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFil
>> + e.Path)] = [item.Target for item in
>> ma.CodaTargetList]
>> + Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}
>> + for Module in ModuleList:
>> # Get ModuleAutoGen object to generate C code file and
>makefile
>> - Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain,
>Arch, self.PlatformFile,Pa.DataPipe)
>> + Ma = ModuleAutoGen(Wa, Module, BuildTarget,
>> + ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
>>
>> - if Ma is None:
>> + if Ma is None:
>> + continue
>> + if Ma.PcdIsDriver:
>> + Ma.PlatformInfo = Pa
>> + Ma.Workspace = Wa
>> + PcdMaList.append(Ma)
>> + TotalModules.append(Ma)
>> + # Initialize all modules in tracking to 'FAIL'
>> + GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
>> +
>> +
>> + mqueue = mp.Queue()
>> + for m in Pa.GetAllModuleInfo:
>> + mqueue.put(m)
>> + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" %
>(str(Pa.Guid),Pa.Arch))
>> + Pa.DataPipe.dump(data_pipe_file)
>> +
>> + autogen_rt, errorcode = self.StartAutoGen(mqueue,
>> + Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR)
>> +
>> + # Skip cache hit modules
>> + if GlobalData.gBinCacheSource:
>> + for Ma in TotalModules:
>> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
>> + GlobalData.gCacheIR[(Ma.MetaFile.Path,
>Ma.Arch)].PreMakeCacheHit:
>> + self.HashSkipModules.append(Ma)
>> + continue
>> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
>> + GlobalData.gCacheIR[(Ma.MetaFile.Path,
>Ma.Arch)].MakeCacheHit:
>> + self.HashSkipModules.append(Ma)
>> continue
>> - if Ma.PcdIsDriver:
>> - Ma.PlatformInfo = Pa
>> - Ma.Workspace = Wa
>> - PcdMaList.append(Ma)
>> - TotalModules.append(Ma)
>> - # Initialize all modules in tracking to 'FAIL'
>> - GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
>> + BuildModules.append(Ma)
>> + else:
>> + BuildModules.extend(TotalModules)
>>
>> - mqueue = mp.Queue()
>> - for m in Pa.GetAllModuleInfo:
>> - mqueue.put(m)
>> - data_pipe_file = os.path.join(Pa.BuildDir,
>"GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
>> - Pa.DataPipe.dump(data_pipe_file)
>> - autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe,
>self.SkipAutoGen, PcdMaList,
>> GlobalData.gCacheIR)
>> + if not autogen_rt:
>> + self.AutoGenMgr.TerminateWorkers()
>> + self.AutoGenMgr.join(1)
>> + raise FatalError(errorcode)
>> + self.AutoGenTime += int(round((time.time() - AutoGenStart)))
>> + AutoGenIdFile =
>os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
>> + with open(AutoGenIdFile,"w") as fw:
>> + fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))
>> + fw.write("BuildDir=%s\n" % Wa.BuildDir)
>> + fw.write("PlatformGuid=%s\n" %
>str(Wa.AutoGenObjectList[0].Guid))
>> + self.Progress.Stop("done!")
>> + return Wa, BuildModules
>>
>> - # Skip cache hit modules
>> - if GlobalData.gBinCacheSource:
>> - for Ma in TotalModules:
>> - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
>> - GlobalData.gCacheIR[(Ma.MetaFile.Path,
>Ma.Arch)].PreMakeCacheHit:
>> - self.HashSkipModules.append(Ma)
>> - continue
>> - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
>> - GlobalData.gCacheIR[(Ma.MetaFile.Path,
>Ma.Arch)].MakeCacheHit:
>> - self.HashSkipModules.append(Ma)
>> - continue
>> - self.BuildModules.append(Ma)
>> + def _MultiThreadBuildPlatform(self):
>> + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE
>auto-generated\n', False)
>> + for BuildTarget in self.BuildTargetList:
>> + GlobalData.gGlobalDefines['TARGET'] = BuildTarget
>> + index = 0
>> + for ToolChain in self.ToolChainList:
>> + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
>> + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
>> + GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
>> + index += 1
>> + ExitFlag = threading.Event()
>> + ExitFlag.clear()
>> + if self.SkipAutoGen:
>> + Wa = self.VerifyAutoGenFiles()
>> + if Wa is None:
>> + Wa, self.BuildModules =
>> + self.PerformAutoGen(BuildTarget,ToolChain)
>> else:
>> - self.BuildModules.extend(TotalModules)
>> -
>> - if not autogen_rt:
>> - self.AutoGenMgr.TerminateWorkers()
>> - self.AutoGenMgr.join(0.1)
>> - raise FatalError(errorcode)
>> - self.AutoGenTime += int(round((time.time() - AutoGenStart)))
>> - self.Progress.Stop("done!")
>> + GlobalData.gAutoGenPhase = True
>> + self.BuildModules = self.SetupMakeSetting(Wa)
>> + else:
>> + Wa, self.BuildModules =
>self.PerformAutoGen(BuildTarget,ToolChain)
>> + Pa = Wa.AutoGenObjectList[0]
>> + GlobalData.gAutoGenPhase = False
>>
>> if GlobalData.gBinCacheSource:
>> EdkLogger.quiet("Total cache hit driver num: %s,
>> cache miss driver num: %s" % (len(set(self.HashSkipModules)),
>> len(set(self.BuildModules))))
>> CacheHitMa = set()
>> CacheNotHitMa = set() @@ -2136,18 +2246,17 @@
>> class Build():
>> # Start task scheduler
>> if not BuildTask.IsOnGoing():
>>
>> BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
>>
>> # in case there's an interruption. we need a full version of
>makefile for platform
>> - Pa.CreateMakeFile(False)
>> +
>> if BuildTask.HasError():
>> self.invalidateHash()
>> EdkLogger.error("build", BUILD_ERROR, "Failed to build
>module", ExtraData=GlobalData.gBuildingModule)
>> self.MakeTime += int(round((time.time() -
>> MakeStart)))
>>
>> MakeContiue = time.time()
>> -
>> #
>> #
>> # All modules have been put in build tasks queue. Tell task
>scheduler
>> # to exit if all tasks are completed
>> #
>> @@ -2175,17 +2284,12 @@ class Build():
>> if (Arch == 'IA32' or Arch == 'ARM') and
>> self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >=
>0x100000000:
>> EdkLogger.error("build",
>> PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to
>larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
>> #
>> # Get Module List
>> #
>> - ModuleList = {}
>> - for Pa in Wa.AutoGenObjectList:
>> - for Ma in Pa.ModuleAutoGenList:
>> - if Ma is None:
>> - continue
>> - if not Ma.IsLibrary:
>> - ModuleList[Ma.Guid.upper()] = Ma
>> + ModuleList = {ma.Guid.upper():ma for ma in
>> + self.BuildModules}
>> +
>> #
>> # Rebase module to the preferred memory address before
>GenFds
>> #
>> MapBuffer = []
>> if self.LoadFixAddress != 0:
>> @@ -2206,33 +2310,17 @@ class Build():
>> self.GenFdsTime += int(round((time.time() - GenFdsStart)))
>> #
>> # Save MAP buffer into MAP file.
>> #
>> self._SaveMapFile(MapBuffer, Wa)
>> + self.CreateGuidedSectionToolsFile(Wa)
>> self.invalidateHash()
>> -
>> ## Generate GuidedSectionTools.txt in the FV directories.
>> #
>> - def CreateGuidedSectionToolsFile(self):
>> + def CreateGuidedSectionToolsFile(self,Wa):
>> for BuildTarget in self.BuildTargetList:
>> for ToolChain in self.ToolChainList:
>> - Wa = WorkspaceAutoGen(
>> - self.WorkspaceDir,
>> - self.PlatformFile,
>> - BuildTarget,
>> - ToolChain,
>> - self.ArchList,
>> - self.BuildDatabase,
>> - self.TargetTxt,
>> - self.ToolDef,
>> - self.Fdf,
>> - self.FdList,
>> - self.FvList,
>> - self.CapList,
>> - self.SkuId,
>> - self.UniFlag
>> - )
>> FvDir = Wa.FvDir
>> if not os.path.exists(FvDir):
>> continue
>>
>> for Arch in self.ArchList:
>> @@ -2285,11 +2373,10 @@ class Build():
>> if not self.SpawnMode or self.Target not in ["", "all"]:
>> self.SpawnMode = False
>> self._BuildPlatform()
>> else:
>> self._MultiThreadBuildPlatform()
>> - self.CreateGuidedSectionToolsFile()
>> else:
>> self.SpawnMode = False
>> self._BuildModule()
>>
>> if self.Target == 'cleanall':
>> --
>> 2.20.1.windows.1
>>
>>
>>
>
>
>
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen
[not found] ` <15C2B2E9C55F0B32.16248@groups.io>
@ 2019-09-09 7:15 ` Liming Gao
0 siblings, 0 replies; 6+ messages in thread
From: Liming Gao @ 2019-09-09 7:15 UTC (permalink / raw)
To: devel@edk2.groups.io, Gao, Liming, Feng, Bob C; +Cc: Shi, Steven
Sorry, should be Acked-by: Liming Gao <liming.gao@intel.com>
>-----Original Message-----
>From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of
>Liming Gao
>Sent: Monday, September 09, 2019 2:59 PM
>To: Feng, Bob C <bob.c.feng@intel.com>; devel@edk2.groups.io
>Cc: Shi, Steven <steven.shi@intel.com>
>Subject: Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for
>IgnoreAutoGen
>
>Liming Gao <liming.gao@intel.com>
>
>>-----Original Message-----
>>From: Feng, Bob C
>>Sent: Thursday, September 05, 2019 10:06 PM
>>To: devel@edk2.groups.io; Feng, Bob C <bob.c.feng@intel.com>; Gao,
>Liming
>><liming.gao@intel.com>
>>Cc: Shi, Steven <steven.shi@intel.com>
>>Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for
>>IgnoreAutoGen
>>
>>Sorry for the incomplete mail.
>>
>>I verified this patch on Ovmf, Emulator, MinPlatforms and Intel Server
>>Platform.
>>
>>Thanks,
>>Bob
>>
>>-----Original Message-----
>>From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Bob
>Feng
>>Sent: Thursday, September 5, 2019 10:04 PM
>>To: Gao, Liming <liming.gao@intel.com>; devel@edk2.groups.io
>>Cc: Shi, Steven <steven.shi@intel.com>
>>Subject: Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for
>>IgnoreAutoGen
>>
>>Liming,
>>
>>Yes, this change only impact -u option.
>>
>>I separate _MultiThreadBuildPlatform function into two functions and create
>2
>>new functions specially for IgnoreAutoGen, so the there looks big change
>for
>>build.py in this patch. I verified this patch on Ovmf, E
>>
>>-Bob
>>
>>-----Original Message-----
>>From: Gao, Liming
>>Sent: Thursday, September 5, 2019 9:12 PM
>>To: devel@edk2.groups.io; Feng, Bob C <bob.c.feng@intel.com>
>>Cc: Shi, Steven <steven.shi@intel.com>
>>Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for
>>IgnoreAutoGen
>>
>>Bob:
>> Does this change only impact -u option behavior?
>>
>>Thanks
>>Liming
>>> -----Original Message-----
>>> From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of
>>> Bob Feng
>>> Sent: Thursday, September 5, 2019 5:05 PM
>>> To: devel@edk2.groups.io
>>> Cc: Gao, Liming <liming.gao@intel.com>; Shi, Steven
>>> <steven.shi@intel.com>; Feng, Bob C <bob.c.feng@intel.com>
>>> Subject: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for
>>> IgnoreAutoGen
>>>
>>> https://bugzilla.tianocore.org/show_bug.cgi?id=2080
>>>
>>> This patch is to improve build -u option to re-use
>>> GlobalVar_<platformguid>_<arch>.bin file which is introduced by
>>> multiple-process-autogen feature.
>>>
>>> Cc: Liming Gao <liming.gao@intel.com>
>>> Cc: Steven Shi <steven.shi@intel.com>
>>> Signed-off-by: Bob Feng <bob.c.feng@intel.com>
>>> ---
>>> .../Source/Python/AutoGen/AutoGenWorker.py | 2 +-
>>> .../Source/Python/AutoGen/ModuleAutoGen.py | 4 +-
>>> .../Python/AutoGen/ModuleAutoGenHelper.py | 29 ++
>>> .../Source/Python/AutoGen/PlatformAutoGen.py | 2 +
>>> .../Source/Python/AutoGen/WorkspaceAutoGen.py | 1 -
>>> BaseTools/Source/Python/build/build.py | 355 +++++++++++-------
>>> 6 files changed, 255 insertions(+), 138 deletions(-)
>>>
>>> diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
>>> b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
>>> index 2e68538b1cb4..f488ae9d5f80 100755
>>> --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
>>> +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
>>> @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process):
>>> Ma.ReferenceModules =
>>Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
>>> if GlobalData.gBinCacheSource and CommandTarget in [None, "",
>>"all"]:
>>> Ma.GenModuleFilesHash(GlobalData.gCacheIR)
>>> Ma.GenPreMakefileHash(GlobalData.gCacheIR)
>>> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
>>> - continue
>>> + continue
>>>
>>> Ma.CreateCodeFile(False)
>>>
>>> Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File,
>>> Ma.Arch),[]))
>>>
>>> if GlobalData.gBinCacheSource and CommandTarget in [None, "",
>>"all"]:
>>> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
>>> b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
>>> index 3bb7e91154ac..5f28681e3146 100755
>>> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
>>> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
>>> @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen):
>>> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
>>> self.GenModuleFilesHash(gDict)
>>>
>>> if not (self.MetaFile.Path, self.Arch) in gDict or \
>>> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
>>> - EdkLogger.quiet("[cache warning]: Cannot generate
>>ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
>>> self.Arch))
>>> - return
>>> + EdkLogger.quiet("[cache warning]: Cannot generate
>>> + ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
>>> self.Arch))
>>> + return
>>>
>>> # Initialze hash object
>>> m = hashlib.md5()
>>>
>>> # Add Platform level hash
>>> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
>>> b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
>>> index c7591253debd..9dd93b9beb12 100644
>>> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
>>> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
>>> @@ -234,22 +234,51 @@ class AutoGenInfo(object):
>>> #
>>> #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in
>>> multiple thread # class WorkSpaceInfo(AutoGenInfo):
>>> def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch):
>>> + if not hasattr(self, "_Init"):
>>> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch)
>>> + self._Init = True
>>> + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch):
>>> self._SrcTimeStamp = 0
>>> self.Db = BuildDB
>>> self.BuildDatabase = self.Db.BuildObject
>>> self.Target = Target
>>> self.ToolChain = ToolChain
>>> self.WorkspaceDir = Workspace
>>> self.ActivePlatform = MetaFile
>>> self.ArchList = Arch
>>> + self.AutoGenObjectList = []
>>> + @property
>>> + def BuildDir(self):
>>> + return self.AutoGenObjectList[0].BuildDir
>>>
>>> + @property
>>> + def Name(self):
>>> + return self.AutoGenObjectList[0].Platform.PlatformName
>>> +
>>> + @property
>>> + def FlashDefinition(self):
>>> + return self.AutoGenObjectList[0].Platform.FlashDefinition
>>> + @property
>>> + def GenFdsCommandDict(self):
>>> + FdsCommandDict =
>>self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict")
>>> + if FdsCommandDict:
>>> + return FdsCommandDict
>>> + return {}
>>> +
>>> + @cached_property
>>> + def FvDir(self):
>>> + return os.path.join(self.BuildDir, TAB_FV_DIRECTORY)
>>>
>>> class PlatformInfo(AutoGenInfo):
>>> def __init__(self, Workspace, MetaFile, Target, ToolChain,
>Arch,DataPipe):
>>> + if not hasattr(self, "_Init"):
>>> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe)
>>> + self._Init = True
>>> + def do_init(self,Workspace, MetaFile, Target, ToolChain,
>Arch,DataPipe):
>>> self.Wa = Workspace
>>> self.WorkspaceDir = self.Wa.WorkspaceDir
>>> self.MetaFile = MetaFile
>>> self.Arch = Arch
>>> self.Target = Target
>>> diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
>>> b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
>>> index 565424a95ead..1e17b6687129 100644
>>> --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
>>> +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
>>> @@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen):
>>> key = (Ma.MetaFile.File, self.Arch)
>>> if key in FfsCommand:
>>> Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key])
>>> else:
>>> Ma.CreateMakeFile(CreateModuleMakeFile)
>>> + self.CreateLibModuelDirs()
>>>
>>> + def CreateLibModuelDirs(self):
>>> # no need to create makefile for the platform more than once
>>> if self.IsMakeFileCreated:
>>> return
>>>
>>> # create library/module build dirs for platform diff --git
>>> a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
>>> b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
>>> index 24942674721f..f9d2c216169c 100644
>>> --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
>>> +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
>>> @@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen):
>>> self._CheckPcdDefineAndType()
>>>
>>> self.CreateBuildOptionsFile()
>>> self.CreatePcdTokenNumberFile()
>>> self.CreateModuleHashInfo()
>>> - GlobalData.gAutoGenPhase = False
>>>
>>> #
>>> # Merge Arch
>>> #
>>> def MergeArch(self):
>>> diff --git a/BaseTools/Source/Python/build/build.py
>>> b/BaseTools/Source/Python/build/build.py
>>> index 0406ac314b65..db57c3e526da 100755
>>> --- a/BaseTools/Source/Python/build/build.py
>>> +++ b/BaseTools/Source/Python/build/build.py
>>> @@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import
>>> PatchBinaryFile
>>>
>>> import Common.GlobalData as GlobalData from GenFds.GenFds import
>>> GenFds, GenFdsApi import multiprocessing as mp from multiprocessing
>>> import Manager
>>> +from AutoGen.DataPipe import MemoryDataPipe from
>>> +AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo
>>from
>>> +GenFds.FdfParser import FdfParser
>>>
>>>
>>> ## standard targets of build command
>>> gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries',
>>> 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
>>>
>>> @@ -837,11 +840,11 @@ class Build():
>>> for PcdMa in PcdMaList:
>>> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
>>> PcdMa.GenModuleFilesHash(share_data)
>>> PcdMa.GenPreMakefileHash(share_data)
>>> if PcdMa.CanSkipbyPreMakefileCache(share_data):
>>> - continue
>>> + continue
>>>
>>> PcdMa.CreateCodeFile(False)
>>> PcdMa.CreateMakeFile(False,GenFfsList =
>>> DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File,
>>> PcdMa.Arch),[]))
>>>
>>> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
>>> @@ -1218,18 +1221,28 @@ class Build():
>>> mqueue = mp.Queue()
>>> for m in AutoGenObject.GetAllModuleInfo:
>>> mqueue.put(m)
>>>
>>> AutoGenObject.DataPipe.DataContainer = {"CommandTarget":
>>> self.Target}
>>> + AutoGenObject.DataPipe.DataContainer =
>{"Workspace_timestamp":
>>AutoGenObject.Workspace._SrcTimeStamp}
>>> + AutoGenObject.CreateLibModuelDirs()
>>> + AutoGenObject.DataPipe.DataContainer =
>>{"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
>>> + AutoGenObject.DataPipe.DataContainer =
>>{"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
>>> + AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict":
>>> + AutoGenObject.Workspace.GenFdsCommandDict}
>>> self.Progress.Start("Generating makefile and code")
>>> data_pipe_file = os.path.join(AutoGenObject.BuildDir,
>>> "GlobalVar_%s_%s.bin" %
>>> (str(AutoGenObject.Guid),AutoGenObject.Arch))
>>> AutoGenObject.DataPipe.dump(data_pipe_file)
>>> autogen_rt,errorcode = self.StartAutoGen(mqueue,
>>> AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList,
>>> GlobalData.gCacheIR)
>>> + AutoGenIdFile =
>>os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
>>> + with open(AutoGenIdFile,"w") as fw:
>>> + fw.write("Arch=%s\n" %
>>"|".join((AutoGenObject.Workspace.ArchList)))
>>> + fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir)
>>> + fw.write("PlatformGuid=%s\n" %
>>> + str(AutoGenObject.Guid))
>>> self.Progress.Stop("done!")
>>> if not autogen_rt:
>>> self.AutoGenMgr.TerminateWorkers()
>>> - self.AutoGenMgr.join(0.1)
>>> + self.AutoGenMgr.join(1)
>>> raise FatalError(errorcode)
>>> AutoGenObject.CreateCodeFile(False)
>>> AutoGenObject.CreateMakeFile(False)
>>> else:
>>> # always recreate top/platform makefile when clean, just
>>> in case of inconsistency @@ -1778,10 +1791,11 @@ class Build():
>>> self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
>>> #
>>> # Save MAP buffer into MAP file.
>>> #
>>> self._SaveMapFile (MapBuffer, Wa)
>>> + self.CreateGuidedSectionToolsFile(Wa)
>>>
>>> ## Build active module for different build targets, different tool chains
>>and different archs
>>> #
>>> def _BuildModule(self):
>>> for BuildTarget in self.BuildTargetList:
>>> @@ -1852,13 +1866,13 @@ class Build():
>>>
>>> if GlobalData.gBinCacheSource and self.Target in [None, "",
>>"all"]:
>>> Ma.GenModuleFilesHash(GlobalData.gCacheIR)
>>> Ma.GenPreMakefileHash(GlobalData.gCacheIR)
>>> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
>>> - self.HashSkipModules.append(Ma)
>>> - EdkLogger.quiet("cache hit: %s[%s]" %
>(Ma.MetaFile.Path,
>>Ma.Arch))
>>> - continue
>>> + self.HashSkipModules.append(Ma)
>>> + EdkLogger.quiet("cache hit: %s[%s]" %
>>(Ma.MetaFile.Path, Ma.Arch))
>>> + continue
>>>
>>> # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run',
>>'fds'
>>> if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
>>> # for target which must generate AutoGen code and
>>makefile
>>> if not self.SkipAutoGen or self.Target == 'genc':
>>> @@ -1987,125 +2001,221 @@ class Build():
>>> GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self,
>>ArchList, GlobalData)
>>> for Cmd in GenFfsDict:
>>> tmpInf, tmpArch = GenFfsDict[Cmd]
>>> CmdSetDict[tmpInf, tmpArch].add(Cmd)
>>> return CmdSetDict
>>> + def VerifyAutoGenFiles(self):
>>> + AutoGenIdFile =
>>os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
>>> + try:
>>> + with open(AutoGenIdFile) as fd:
>>> + lines = fd.readlines()
>>> + except:
>>> + return None
>>> + for line in lines:
>>> + if "Arch" in line:
>>> + ArchList = line.strip().split("=")[1].split("|")
>>> + if "BuildDir" in line:
>>> + BuildDir = line.split("=")[1].strip()
>>> + if "PlatformGuid" in line:
>>> + PlatformGuid = line.split("=")[1].strip()
>>> + GlobalVarList = []
>>> + for arch in ArchList:
>>> + global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" %
>>(str(PlatformGuid),arch))
>>> + if not os.path.exists(global_var):
>>> + return None
>>> + GlobalVarList.append(global_var)
>>> + for global_var in GlobalVarList:
>>> + data_pipe = MemoryDataPipe()
>>> + data_pipe.load(global_var)
>>> + target = data_pipe.Get("P_Info").get("Target")
>>> + toolchain = data_pipe.Get("P_Info").get("ToolChain")
>>> + archlist = data_pipe.Get("P_Info").get("ArchList")
>>> + Arch = data_pipe.Get("P_Info").get("Arch")
>>> + active_p = data_pipe.Get("P_Info").get("ActivePlatform")
>>> + workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir")
>>> + PackagesPath = os.getenv("PACKAGES_PATH")
>>> + mws.setWs(workspacedir, PackagesPath)
>>> + LibraryBuildDirectoryList =
>data_pipe.Get("LibraryBuildDirectoryList")
>>> + ModuleBuildDirectoryList =
>>> + data_pipe.Get("ModuleBuildDirectoryList")
>>> +
>>> + for m_build_dir in LibraryBuildDirectoryList:
>>> + if not
>>os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[G
>e
>>nMake.gMakeType])):
>>> + return None
>>> + for m_build_dir in ModuleBuildDirectoryList:
>>> + if not
>>os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[G
>e
>>nMake.gMakeType])):
>>> + return None
>>> + Wa = WorkSpaceInfo(
>>> + workspacedir,active_p,target,toolchain,archlist
>>> + )
>>> + Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe)
>>> + Wa.AutoGenObjectList.append(Pa)
>>> + return Wa
>>> + def SetupMakeSetting(self,Wa):
>>> + BuildModules = []
>>> + for Pa in Wa.AutoGenObjectList:
>>> + for m in Pa._MbList:
>>> + ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget,
>>Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe)
>>> + BuildModules.append(ma)
>>> + fdf_file = Wa.FlashDefinition
>>> + if fdf_file:
>>> + Fdf = FdfParser(fdf_file.Path)
>>> + Fdf.ParseFile()
>>> + GlobalData.gFdfParser = Fdf
>>> + if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:
>>> + FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]
>>> + for FdRegion in FdDict.RegionList:
>>> + if str(FdRegion.RegionType) is 'FILE' and
>>self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):
>>> + if int(FdRegion.Offset) % 8 != 0:
>>> + EdkLogger.error("build", FORMAT_INVALID,
>>> + 'The VPD Base Address %s must be 8-byte aligned.' %
>>> (FdRegion.Offset))
>>> + Wa.FdfProfile = Fdf.Profile
>>> + self.Fdf = Fdf
>>> + else:
>>> + self.Fdf = None
>>> + return BuildModules
>>>
>>> ## Build a platform in multi-thread mode
>>> #
>>> - def _MultiThreadBuildPlatform(self):
>>> - SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE
>>auto-generated\n', False)
>>> - for BuildTarget in self.BuildTargetList:
>>> - GlobalData.gGlobalDefines['TARGET'] = BuildTarget
>>> - index = 0
>>> - for ToolChain in self.ToolChainList:
>>> - WorkspaceAutoGenTime = time.time()
>>> - GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
>>> - GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
>>> - GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
>>> - index += 1
>>> - Wa = WorkspaceAutoGen(
>>> - self.WorkspaceDir,
>>> - self.PlatformFile,
>>> - BuildTarget,
>>> - ToolChain,
>>> - self.ArchList,
>>> - self.BuildDatabase,
>>> - self.TargetTxt,
>>> - self.ToolDef,
>>> - self.Fdf,
>>> - self.FdList,
>>> - self.FvList,
>>> - self.CapList,
>>> - self.SkuId,
>>> - self.UniFlag,
>>> - self.Progress
>>> - )
>>> - self.Fdf = Wa.FdfFile
>>> - self.LoadFixAddress = Wa.Platform.LoadFixAddress
>>> - self.BuildReport.AddPlatformReport(Wa)
>>> - Wa.CreateMakeFile(False)
>>> + def PerformAutoGen(self,BuildTarget,ToolChain):
>>> + WorkspaceAutoGenTime = time.time()
>>> + Wa = WorkspaceAutoGen(
>>> + self.WorkspaceDir,
>>> + self.PlatformFile,
>>> + BuildTarget,
>>> + ToolChain,
>>> + self.ArchList,
>>> + self.BuildDatabase,
>>> + self.TargetTxt,
>>> + self.ToolDef,
>>> + self.Fdf,
>>> + self.FdList,
>>> + self.FvList,
>>> + self.CapList,
>>> + self.SkuId,
>>> + self.UniFlag,
>>> + self.Progress
>>> + )
>>> + self.Fdf = Wa.FdfFile
>>> + self.LoadFixAddress = Wa.Platform.LoadFixAddress
>>> + self.BuildReport.AddPlatformReport(Wa)
>>> + Wa.CreateMakeFile(False)
>>>
>>> # Add ffs build to makefile
>>> - CmdListDict = {}
>>> - if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
>>> - CmdListDict = self._GenFfsCmd(Wa.ArchList)
>>> + CmdListDict = {}
>>> + if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
>>> + CmdListDict = self._GenFfsCmd(Wa.ArchList)
>>>
>>> - # Add Platform and Package level hash in share_data for module
>>hash calculation later
>>> - if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
>>> - GlobalData.gCacheIR[('PlatformHash')] =
>>GlobalData.gPlatformHash
>>> - for PkgName in GlobalData.gPackageHash.keys():
>>> - GlobalData.gCacheIR[(PkgName, 'PackageHash')] =
>>GlobalData.gPackageHash[PkgName]
>>> + # Add Platform and Package level hash in share_data for module hash
>>calculation later
>>> + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
>>> + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
>>> + for PkgName in GlobalData.gPackageHash.keys():
>>> + GlobalData.gCacheIR[(PkgName, 'PackageHash')] =
>>> + GlobalData.gPackageHash[PkgName]
>>>
>>> - # multi-thread exit flag
>>> - ExitFlag = threading.Event()
>>> - ExitFlag.clear()
>>> - self.AutoGenTime += int(round((time.time() -
>>WorkspaceAutoGenTime)))
>>> - self.BuildModules = []
>>> - TotalModules = []
>>> - for Arch in Wa.ArchList:
>>> - PcdMaList = []
>>> - AutoGenStart = time.time()
>>> - GlobalData.gGlobalDefines['ARCH'] = Arch
>>> - Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget,
>>ToolChain, Arch)
>>> - if Pa is None:
>>> - continue
>>> - ModuleList = []
>>> - for Inf in Pa.Platform.Modules:
>>> - ModuleList.append(Inf)
>>> + self.AutoGenTime += int(round((time.time() -
>>WorkspaceAutoGenTime)))
>>> + BuildModules = []
>>> + TotalModules = []
>>> + for Arch in Wa.ArchList:
>>> + PcdMaList = []
>>> + AutoGenStart = time.time()
>>> + GlobalData.gGlobalDefines['ARCH'] = Arch
>>> + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain,
>>Arch)
>>> + if Pa is None:
>>> + continue
>>> + ModuleList = []
>>> + for Inf in Pa.Platform.Modules:
>>> + ModuleList.append(Inf)
>>> # Add the INF only list in FDF
>>> - if GlobalData.gFdfParser is not None:
>>> - for InfName in GlobalData.gFdfParser.Profile.InfList:
>>> - Inf = PathClass(NormPath(InfName), self.WorkspaceDir,
>Arch)
>>> - if Inf in Pa.Platform.Modules:
>>> - continue
>>> - ModuleList.append(Inf)
>>> - Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
>>> - Pa.DataPipe.DataContainer = {"Workspace_timestamp":
>>Wa._SrcTimeStamp}
>>> - Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
>>> - for Module in ModuleList:
>>> + if GlobalData.gFdfParser is not None:
>>> + for InfName in GlobalData.gFdfParser.Profile.InfList:
>>> + Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
>>> + if Inf in Pa.Platform.Modules:
>>> + continue
>>> + ModuleList.append(Inf)
>>> + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
>>> + Pa.DataPipe.DataContainer = {"Workspace_timestamp":
>>Wa._SrcTimeStamp}
>>> + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
>>> + Pa.CreateLibModuelDirs()
>>> + Pa.DataPipe.DataContainer =
>>{"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}
>>> + Pa.DataPipe.DataContainer =
>>{"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}
>>> + Pa.DataPipe.DataContainer = {"FdsCommandDict":
>>Wa.GenFdsCommandDict}
>>> + ModuleCodaFile = {}
>>> + for ma in Pa.ModuleAutoGenList:
>>> +
>>> +
>ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFil
>>> + e.Path)] = [item.Target for item in
>>> ma.CodaTargetList]
>>> + Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}
>>> + for Module in ModuleList:
>>> # Get ModuleAutoGen object to generate C code file and
>>makefile
>>> - Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain,
>>Arch, self.PlatformFile,Pa.DataPipe)
>>> + Ma = ModuleAutoGen(Wa, Module, BuildTarget,
>>> + ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
>>>
>>> - if Ma is None:
>>> + if Ma is None:
>>> + continue
>>> + if Ma.PcdIsDriver:
>>> + Ma.PlatformInfo = Pa
>>> + Ma.Workspace = Wa
>>> + PcdMaList.append(Ma)
>>> + TotalModules.append(Ma)
>>> + # Initialize all modules in tracking to 'FAIL'
>>> + GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
>>> +
>>> +
>>> + mqueue = mp.Queue()
>>> + for m in Pa.GetAllModuleInfo:
>>> + mqueue.put(m)
>>> + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" %
>>(str(Pa.Guid),Pa.Arch))
>>> + Pa.DataPipe.dump(data_pipe_file)
>>> +
>>> + autogen_rt, errorcode = self.StartAutoGen(mqueue,
>>> + Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR)
>>> +
>>> + # Skip cache hit modules
>>> + if GlobalData.gBinCacheSource:
>>> + for Ma in TotalModules:
>>> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
>>> + GlobalData.gCacheIR[(Ma.MetaFile.Path,
>>Ma.Arch)].PreMakeCacheHit:
>>> + self.HashSkipModules.append(Ma)
>>> + continue
>>> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
>>> + GlobalData.gCacheIR[(Ma.MetaFile.Path,
>>Ma.Arch)].MakeCacheHit:
>>> + self.HashSkipModules.append(Ma)
>>> continue
>>> - if Ma.PcdIsDriver:
>>> - Ma.PlatformInfo = Pa
>>> - Ma.Workspace = Wa
>>> - PcdMaList.append(Ma)
>>> - TotalModules.append(Ma)
>>> - # Initialize all modules in tracking to 'FAIL'
>>> - GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
>>> + BuildModules.append(Ma)
>>> + else:
>>> + BuildModules.extend(TotalModules)
>>>
>>> - mqueue = mp.Queue()
>>> - for m in Pa.GetAllModuleInfo:
>>> - mqueue.put(m)
>>> - data_pipe_file = os.path.join(Pa.BuildDir,
>>"GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
>>> - Pa.DataPipe.dump(data_pipe_file)
>>> - autogen_rt, errorcode = self.StartAutoGen(mqueue,
>Pa.DataPipe,
>>self.SkipAutoGen, PcdMaList,
>>> GlobalData.gCacheIR)
>>> + if not autogen_rt:
>>> + self.AutoGenMgr.TerminateWorkers()
>>> + self.AutoGenMgr.join(1)
>>> + raise FatalError(errorcode)
>>> + self.AutoGenTime += int(round((time.time() - AutoGenStart)))
>>> + AutoGenIdFile =
>>os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
>>> + with open(AutoGenIdFile,"w") as fw:
>>> + fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))
>>> + fw.write("BuildDir=%s\n" % Wa.BuildDir)
>>> + fw.write("PlatformGuid=%s\n" %
>>str(Wa.AutoGenObjectList[0].Guid))
>>> + self.Progress.Stop("done!")
>>> + return Wa, BuildModules
>>>
>>> - # Skip cache hit modules
>>> - if GlobalData.gBinCacheSource:
>>> - for Ma in TotalModules:
>>> - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and
>\
>>> - GlobalData.gCacheIR[(Ma.MetaFile.Path,
>>Ma.Arch)].PreMakeCacheHit:
>>> - self.HashSkipModules.append(Ma)
>>> - continue
>>> - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and
>\
>>> - GlobalData.gCacheIR[(Ma.MetaFile.Path,
>>Ma.Arch)].MakeCacheHit:
>>> - self.HashSkipModules.append(Ma)
>>> - continue
>>> - self.BuildModules.append(Ma)
>>> + def _MultiThreadBuildPlatform(self):
>>> + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE
>>auto-generated\n', False)
>>> + for BuildTarget in self.BuildTargetList:
>>> + GlobalData.gGlobalDefines['TARGET'] = BuildTarget
>>> + index = 0
>>> + for ToolChain in self.ToolChainList:
>>> + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
>>> + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
>>> + GlobalData.gGlobalDefines['FAMILY'] =
>self.ToolChainFamily[index]
>>> + index += 1
>>> + ExitFlag = threading.Event()
>>> + ExitFlag.clear()
>>> + if self.SkipAutoGen:
>>> + Wa = self.VerifyAutoGenFiles()
>>> + if Wa is None:
>>> + Wa, self.BuildModules =
>>> + self.PerformAutoGen(BuildTarget,ToolChain)
>>> else:
>>> - self.BuildModules.extend(TotalModules)
>>> -
>>> - if not autogen_rt:
>>> - self.AutoGenMgr.TerminateWorkers()
>>> - self.AutoGenMgr.join(0.1)
>>> - raise FatalError(errorcode)
>>> - self.AutoGenTime += int(round((time.time() - AutoGenStart)))
>>> - self.Progress.Stop("done!")
>>> + GlobalData.gAutoGenPhase = True
>>> + self.BuildModules = self.SetupMakeSetting(Wa)
>>> + else:
>>> + Wa, self.BuildModules =
>>self.PerformAutoGen(BuildTarget,ToolChain)
>>> + Pa = Wa.AutoGenObjectList[0]
>>> + GlobalData.gAutoGenPhase = False
>>>
>>> if GlobalData.gBinCacheSource:
>>> EdkLogger.quiet("Total cache hit driver num: %s,
>>> cache miss driver num: %s" % (len(set(self.HashSkipModules)),
>>> len(set(self.BuildModules))))
>>> CacheHitMa = set()
>>> CacheNotHitMa = set() @@ -2136,18 +2246,17 @@
>>> class Build():
>>> # Start task scheduler
>>> if not BuildTask.IsOnGoing():
>>>
>>> BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
>>>
>>> # in case there's an interruption. we need a full version of
>>makefile for platform
>>> - Pa.CreateMakeFile(False)
>>> +
>>> if BuildTask.HasError():
>>> self.invalidateHash()
>>> EdkLogger.error("build", BUILD_ERROR, "Failed to build
>>module", ExtraData=GlobalData.gBuildingModule)
>>> self.MakeTime += int(round((time.time() -
>>> MakeStart)))
>>>
>>> MakeContiue = time.time()
>>> -
>>> #
>>> #
>>> # All modules have been put in build tasks queue. Tell task
>>scheduler
>>> # to exit if all tasks are completed
>>> #
>>> @@ -2175,17 +2284,12 @@ class Build():
>>> if (Arch == 'IA32' or Arch == 'ARM') and
>>> self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >=
>>0x100000000:
>>> EdkLogger.error("build",
>>> PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set
>to
>>larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
>>> #
>>> # Get Module List
>>> #
>>> - ModuleList = {}
>>> - for Pa in Wa.AutoGenObjectList:
>>> - for Ma in Pa.ModuleAutoGenList:
>>> - if Ma is None:
>>> - continue
>>> - if not Ma.IsLibrary:
>>> - ModuleList[Ma.Guid.upper()] = Ma
>>> + ModuleList = {ma.Guid.upper():ma for ma in
>>> + self.BuildModules}
>>> +
>>> #
>>> # Rebase module to the preferred memory address before
>>GenFds
>>> #
>>> MapBuffer = []
>>> if self.LoadFixAddress != 0:
>>> @@ -2206,33 +2310,17 @@ class Build():
>>> self.GenFdsTime += int(round((time.time() - GenFdsStart)))
>>> #
>>> # Save MAP buffer into MAP file.
>>> #
>>> self._SaveMapFile(MapBuffer, Wa)
>>> + self.CreateGuidedSectionToolsFile(Wa)
>>> self.invalidateHash()
>>> -
>>> ## Generate GuidedSectionTools.txt in the FV directories.
>>> #
>>> - def CreateGuidedSectionToolsFile(self):
>>> + def CreateGuidedSectionToolsFile(self,Wa):
>>> for BuildTarget in self.BuildTargetList:
>>> for ToolChain in self.ToolChainList:
>>> - Wa = WorkspaceAutoGen(
>>> - self.WorkspaceDir,
>>> - self.PlatformFile,
>>> - BuildTarget,
>>> - ToolChain,
>>> - self.ArchList,
>>> - self.BuildDatabase,
>>> - self.TargetTxt,
>>> - self.ToolDef,
>>> - self.Fdf,
>>> - self.FdList,
>>> - self.FvList,
>>> - self.CapList,
>>> - self.SkuId,
>>> - self.UniFlag
>>> - )
>>> FvDir = Wa.FvDir
>>> if not os.path.exists(FvDir):
>>> continue
>>>
>>> for Arch in self.ArchList:
>>> @@ -2285,11 +2373,10 @@ class Build():
>>> if not self.SpawnMode or self.Target not in ["", "all"]:
>>> self.SpawnMode = False
>>> self._BuildPlatform()
>>> else:
>>> self._MultiThreadBuildPlatform()
>>> - self.CreateGuidedSectionToolsFile()
>>> else:
>>> self.SpawnMode = False
>>> self._BuildModule()
>>>
>>> if self.Target == 'cleanall':
>>> --
>>> 2.20.1.windows.1
>>>
>>>
>>>
>>
>>
>>
>
>
>
^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2019-09-09 7:15 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2019-09-05 9:04 [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen Bob Feng
2019-09-05 13:11 ` [edk2-devel] " Liming Gao
2019-09-05 14:04 ` Bob Feng
[not found] ` <15C18FCC38905317.10639@groups.io>
2019-09-05 14:05 ` Bob Feng
2019-09-09 6:58 ` Liming Gao
[not found] ` <15C2B2E9C55F0B32.16248@groups.io>
2019-09-09 7:15 ` Liming Gao
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox