* [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen @ 2019-09-05 9:04 Bob Feng 2019-09-05 13:11 ` [edk2-devel] " Liming Gao 0 siblings, 1 reply; 6+ messages in thread From: Bob Feng @ 2019-09-05 9:04 UTC (permalink / raw) To: devel; +Cc: Liming Gao, Steven Shi, Bob Feng https://bugzilla.tianocore.org/show_bug.cgi?id=2080 This patch is to improve build -u option to re-use GlobalVar_<platformguid>_<arch>.bin file which is introduced by multiple-process-autogen feature. Cc: Liming Gao <liming.gao@intel.com> Cc: Steven Shi <steven.shi@intel.com> Signed-off-by: Bob Feng <bob.c.feng@intel.com> --- .../Source/Python/AutoGen/AutoGenWorker.py | 2 +- .../Source/Python/AutoGen/ModuleAutoGen.py | 4 +- .../Python/AutoGen/ModuleAutoGenHelper.py | 29 ++ .../Source/Python/AutoGen/PlatformAutoGen.py | 2 + .../Source/Python/AutoGen/WorkspaceAutoGen.py | 1 - BaseTools/Source/Python/build/build.py | 355 +++++++++++------- 6 files changed, 255 insertions(+), 138 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py index 2e68538b1cb4..f488ae9d5f80 100755 --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process): Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: Ma.GenModuleFilesHash(GlobalData.gCacheIR) Ma.GenPreMakefileHash(GlobalData.gCacheIR) if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): - continue + continue Ma.CreateCodeFile(False) Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[])) if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py index 3bb7e91154ac..5f28681e3146 100755 --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen): not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: self.GenModuleFilesHash(gDict) if not (self.MetaFile.Path, self.Arch) in gDict or \ not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: - EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, self.Arch)) - return + EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, self.Arch)) + return # Initialze hash object m = hashlib.md5() # Add Platform level hash diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py index c7591253debd..9dd93b9beb12 100644 --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py @@ -234,22 +234,51 @@ class AutoGenInfo(object): # #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in multiple thread # class WorkSpaceInfo(AutoGenInfo): def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch): + if not hasattr(self, "_Init"): + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch) + self._Init = True + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch): self._SrcTimeStamp = 0 self.Db = BuildDB self.BuildDatabase = self.Db.BuildObject self.Target = Target self.ToolChain = ToolChain self.WorkspaceDir = Workspace self.ActivePlatform = MetaFile self.ArchList = Arch + self.AutoGenObjectList = [] + @property + def BuildDir(self): + return self.AutoGenObjectList[0].BuildDir + @property + def Name(self): + return self.AutoGenObjectList[0].Platform.PlatformName + + @property + def FlashDefinition(self): + return self.AutoGenObjectList[0].Platform.FlashDefinition + @property + def GenFdsCommandDict(self): + FdsCommandDict = self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict") + if FdsCommandDict: + return FdsCommandDict + return {} + + @cached_property + def FvDir(self): + return os.path.join(self.BuildDir, TAB_FV_DIRECTORY) class PlatformInfo(AutoGenInfo): def __init__(self, Workspace, MetaFile, Target, ToolChain, Arch,DataPipe): + if not hasattr(self, "_Init"): + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe) + self._Init = True + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe): self.Wa = Workspace self.WorkspaceDir = self.Wa.WorkspaceDir self.MetaFile = MetaFile self.Arch = Arch self.Target = Target diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py index 565424a95ead..1e17b6687129 100644 --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py @@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen): key = (Ma.MetaFile.File, self.Arch) if key in FfsCommand: Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key]) else: Ma.CreateMakeFile(CreateModuleMakeFile) + self.CreateLibModuelDirs() + def CreateLibModuelDirs(self): # no need to create makefile for the platform more than once if self.IsMakeFileCreated: return # create library/module build dirs for platform diff --git a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py index 24942674721f..f9d2c216169c 100644 --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py @@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen): self._CheckPcdDefineAndType() self.CreateBuildOptionsFile() self.CreatePcdTokenNumberFile() self.CreateModuleHashInfo() - GlobalData.gAutoGenPhase = False # # Merge Arch # def MergeArch(self): diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index 0406ac314b65..db57c3e526da 100755 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import PatchBinaryFile import Common.GlobalData as GlobalData from GenFds.GenFds import GenFds, GenFdsApi import multiprocessing as mp from multiprocessing import Manager +from AutoGen.DataPipe import MemoryDataPipe +from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo +from GenFds.FdfParser import FdfParser ## standard targets of build command gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run'] @@ -837,11 +840,11 @@ class Build(): for PcdMa in PcdMaList: if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: PcdMa.GenModuleFilesHash(share_data) PcdMa.GenPreMakefileHash(share_data) if PcdMa.CanSkipbyPreMakefileCache(share_data): - continue + continue PcdMa.CreateCodeFile(False) PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[])) if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: @@ -1218,18 +1221,28 @@ class Build(): mqueue = mp.Queue() for m in AutoGenObject.GetAllModuleInfo: mqueue.put(m) AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target} + AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp} + AutoGenObject.CreateLibModuelDirs() + AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList} + AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList} + AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": AutoGenObject.Workspace.GenFdsCommandDict} self.Progress.Start("Generating makefile and code") data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch)) AutoGenObject.DataPipe.dump(data_pipe_file) autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR) + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") + with open(AutoGenIdFile,"w") as fw: + fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList))) + fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir) + fw.write("PlatformGuid=%s\n" % str(AutoGenObject.Guid)) self.Progress.Stop("done!") if not autogen_rt: self.AutoGenMgr.TerminateWorkers() - self.AutoGenMgr.join(0.1) + self.AutoGenMgr.join(1) raise FatalError(errorcode) AutoGenObject.CreateCodeFile(False) AutoGenObject.CreateMakeFile(False) else: # always recreate top/platform makefile when clean, just in case of inconsistency @@ -1778,10 +1791,11 @@ class Build(): self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList) # # Save MAP buffer into MAP file. # self._SaveMapFile (MapBuffer, Wa) + self.CreateGuidedSectionToolsFile(Wa) ## Build active module for different build targets, different tool chains and different archs # def _BuildModule(self): for BuildTarget in self.BuildTargetList: @@ -1852,13 +1866,13 @@ class Build(): if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: Ma.GenModuleFilesHash(GlobalData.gCacheIR) Ma.GenPreMakefileHash(GlobalData.gCacheIR) if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): - self.HashSkipModules.append(Ma) - EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) - continue + self.HashSkipModules.append(Ma) + EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) + continue # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds' if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile if not self.SkipAutoGen or self.Target == 'genc': @@ -1987,125 +2001,221 @@ class Build(): GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData) for Cmd in GenFfsDict: tmpInf, tmpArch = GenFfsDict[Cmd] CmdSetDict[tmpInf, tmpArch].add(Cmd) return CmdSetDict + def VerifyAutoGenFiles(self): + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") + try: + with open(AutoGenIdFile) as fd: + lines = fd.readlines() + except: + return None + for line in lines: + if "Arch" in line: + ArchList = line.strip().split("=")[1].split("|") + if "BuildDir" in line: + BuildDir = line.split("=")[1].strip() + if "PlatformGuid" in line: + PlatformGuid = line.split("=")[1].strip() + GlobalVarList = [] + for arch in ArchList: + global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch)) + if not os.path.exists(global_var): + return None + GlobalVarList.append(global_var) + for global_var in GlobalVarList: + data_pipe = MemoryDataPipe() + data_pipe.load(global_var) + target = data_pipe.Get("P_Info").get("Target") + toolchain = data_pipe.Get("P_Info").get("ToolChain") + archlist = data_pipe.Get("P_Info").get("ArchList") + Arch = data_pipe.Get("P_Info").get("Arch") + active_p = data_pipe.Get("P_Info").get("ActivePlatform") + workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir") + PackagesPath = os.getenv("PACKAGES_PATH") + mws.setWs(workspacedir, PackagesPath) + LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList") + ModuleBuildDirectoryList = data_pipe.Get("ModuleBuildDirectoryList") + + for m_build_dir in LibraryBuildDirectoryList: + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])): + return None + for m_build_dir in ModuleBuildDirectoryList: + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])): + return None + Wa = WorkSpaceInfo( + workspacedir,active_p,target,toolchain,archlist + ) + Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe) + Wa.AutoGenObjectList.append(Pa) + return Wa + def SetupMakeSetting(self,Wa): + BuildModules = [] + for Pa in Wa.AutoGenObjectList: + for m in Pa._MbList: + ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe) + BuildModules.append(ma) + fdf_file = Wa.FlashDefinition + if fdf_file: + Fdf = FdfParser(fdf_file.Path) + Fdf.ParseFile() + GlobalData.gFdfParser = Fdf + if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict: + FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName] + for FdRegion in FdDict.RegionList: + if str(FdRegion.RegionType) is 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList): + if int(FdRegion.Offset) % 8 != 0: + EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' % (FdRegion.Offset)) + Wa.FdfProfile = Fdf.Profile + self.Fdf = Fdf + else: + self.Fdf = None + return BuildModules ## Build a platform in multi-thread mode # - def _MultiThreadBuildPlatform(self): - SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False) - for BuildTarget in self.BuildTargetList: - GlobalData.gGlobalDefines['TARGET'] = BuildTarget - index = 0 - for ToolChain in self.ToolChainList: - WorkspaceAutoGenTime = time.time() - GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain - GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain - GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index] - index += 1 - Wa = WorkspaceAutoGen( - self.WorkspaceDir, - self.PlatformFile, - BuildTarget, - ToolChain, - self.ArchList, - self.BuildDatabase, - self.TargetTxt, - self.ToolDef, - self.Fdf, - self.FdList, - self.FvList, - self.CapList, - self.SkuId, - self.UniFlag, - self.Progress - ) - self.Fdf = Wa.FdfFile - self.LoadFixAddress = Wa.Platform.LoadFixAddress - self.BuildReport.AddPlatformReport(Wa) - Wa.CreateMakeFile(False) + def PerformAutoGen(self,BuildTarget,ToolChain): + WorkspaceAutoGenTime = time.time() + Wa = WorkspaceAutoGen( + self.WorkspaceDir, + self.PlatformFile, + BuildTarget, + ToolChain, + self.ArchList, + self.BuildDatabase, + self.TargetTxt, + self.ToolDef, + self.Fdf, + self.FdList, + self.FvList, + self.CapList, + self.SkuId, + self.UniFlag, + self.Progress + ) + self.Fdf = Wa.FdfFile + self.LoadFixAddress = Wa.Platform.LoadFixAddress + self.BuildReport.AddPlatformReport(Wa) + Wa.CreateMakeFile(False) # Add ffs build to makefile - CmdListDict = {} - if GlobalData.gEnableGenfdsMultiThread and self.Fdf: - CmdListDict = self._GenFfsCmd(Wa.ArchList) + CmdListDict = {} + if GlobalData.gEnableGenfdsMultiThread and self.Fdf: + CmdListDict = self._GenFfsCmd(Wa.ArchList) - # Add Platform and Package level hash in share_data for module hash calculation later - if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: - GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash - for PkgName in GlobalData.gPackageHash.keys(): - GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName] + # Add Platform and Package level hash in share_data for module hash calculation later + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash + for PkgName in GlobalData.gPackageHash.keys(): + GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName] - # multi-thread exit flag - ExitFlag = threading.Event() - ExitFlag.clear() - self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) - self.BuildModules = [] - TotalModules = [] - for Arch in Wa.ArchList: - PcdMaList = [] - AutoGenStart = time.time() - GlobalData.gGlobalDefines['ARCH'] = Arch - Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) - if Pa is None: - continue - ModuleList = [] - for Inf in Pa.Platform.Modules: - ModuleList.append(Inf) + self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) + BuildModules = [] + TotalModules = [] + for Arch in Wa.ArchList: + PcdMaList = [] + AutoGenStart = time.time() + GlobalData.gGlobalDefines['ARCH'] = Arch + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) + if Pa is None: + continue + ModuleList = [] + for Inf in Pa.Platform.Modules: + ModuleList.append(Inf) # Add the INF only list in FDF - if GlobalData.gFdfParser is not None: - for InfName in GlobalData.gFdfParser.Profile.InfList: - Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch) - if Inf in Pa.Platform.Modules: - continue - ModuleList.append(Inf) - Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} - Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} - Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} - for Module in ModuleList: + if GlobalData.gFdfParser is not None: + for InfName in GlobalData.gFdfParser.Profile.InfList: + Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch) + if Inf in Pa.Platform.Modules: + continue + ModuleList.append(Inf) + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} + Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} + Pa.CreateLibModuelDirs() + Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList} + Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList} + Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict} + ModuleCodaFile = {} + for ma in Pa.ModuleAutoGenList: + ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in ma.CodaTargetList] + Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile} + for Module in ModuleList: # Get ModuleAutoGen object to generate C code file and makefile - Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) + Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) - if Ma is None: + if Ma is None: + continue + if Ma.PcdIsDriver: + Ma.PlatformInfo = Pa + Ma.Workspace = Wa + PcdMaList.append(Ma) + TotalModules.append(Ma) + # Initialize all modules in tracking to 'FAIL' + GlobalData.gModuleBuildTracking[Ma] = 'FAIL' + + + mqueue = mp.Queue() + for m in Pa.GetAllModuleInfo: + mqueue.put(m) + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) + Pa.DataPipe.dump(data_pipe_file) + + autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR) + + # Skip cache hit modules + if GlobalData.gBinCacheSource: + for Ma in TotalModules: + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit: + self.HashSkipModules.append(Ma) + continue + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit: + self.HashSkipModules.append(Ma) continue - if Ma.PcdIsDriver: - Ma.PlatformInfo = Pa - Ma.Workspace = Wa - PcdMaList.append(Ma) - TotalModules.append(Ma) - # Initialize all modules in tracking to 'FAIL' - GlobalData.gModuleBuildTracking[Ma] = 'FAIL' + BuildModules.append(Ma) + else: + BuildModules.extend(TotalModules) - mqueue = mp.Queue() - for m in Pa.GetAllModuleInfo: - mqueue.put(m) - data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) - Pa.DataPipe.dump(data_pipe_file) - autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR) + if not autogen_rt: + self.AutoGenMgr.TerminateWorkers() + self.AutoGenMgr.join(1) + raise FatalError(errorcode) + self.AutoGenTime += int(round((time.time() - AutoGenStart))) + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") + with open(AutoGenIdFile,"w") as fw: + fw.write("Arch=%s\n" % "|".join((Wa.ArchList))) + fw.write("BuildDir=%s\n" % Wa.BuildDir) + fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid)) + self.Progress.Stop("done!") + return Wa, BuildModules - # Skip cache hit modules - if GlobalData.gBinCacheSource: - for Ma in TotalModules: - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit: - self.HashSkipModules.append(Ma) - continue - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit: - self.HashSkipModules.append(Ma) - continue - self.BuildModules.append(Ma) + def _MultiThreadBuildPlatform(self): + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False) + for BuildTarget in self.BuildTargetList: + GlobalData.gGlobalDefines['TARGET'] = BuildTarget + index = 0 + for ToolChain in self.ToolChainList: + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain + GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index] + index += 1 + ExitFlag = threading.Event() + ExitFlag.clear() + if self.SkipAutoGen: + Wa = self.VerifyAutoGenFiles() + if Wa is None: + Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain) else: - self.BuildModules.extend(TotalModules) - - if not autogen_rt: - self.AutoGenMgr.TerminateWorkers() - self.AutoGenMgr.join(0.1) - raise FatalError(errorcode) - self.AutoGenTime += int(round((time.time() - AutoGenStart))) - self.Progress.Stop("done!") + GlobalData.gAutoGenPhase = True + self.BuildModules = self.SetupMakeSetting(Wa) + else: + Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain) + Pa = Wa.AutoGenObjectList[0] + GlobalData.gAutoGenPhase = False if GlobalData.gBinCacheSource: EdkLogger.quiet("Total cache hit driver num: %s, cache miss driver num: %s" % (len(set(self.HashSkipModules)), len(set(self.BuildModules)))) CacheHitMa = set() CacheNotHitMa = set() @@ -2136,18 +2246,17 @@ class Build(): # Start task scheduler if not BuildTask.IsOnGoing(): BuildTask.StartScheduler(self.ThreadNumber, ExitFlag) # in case there's an interruption. we need a full version of makefile for platform - Pa.CreateMakeFile(False) + if BuildTask.HasError(): self.invalidateHash() EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) self.MakeTime += int(round((time.time() - MakeStart))) MakeContiue = time.time() - # # # All modules have been put in build tasks queue. Tell task scheduler # to exit if all tasks are completed # @@ -2175,17 +2284,12 @@ class Build(): if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000: EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules") # # Get Module List # - ModuleList = {} - for Pa in Wa.AutoGenObjectList: - for Ma in Pa.ModuleAutoGenList: - if Ma is None: - continue - if not Ma.IsLibrary: - ModuleList[Ma.Guid.upper()] = Ma + ModuleList = {ma.Guid.upper():ma for ma in self.BuildModules} + # # Rebase module to the preferred memory address before GenFds # MapBuffer = [] if self.LoadFixAddress != 0: @@ -2206,33 +2310,17 @@ class Build(): self.GenFdsTime += int(round((time.time() - GenFdsStart))) # # Save MAP buffer into MAP file. # self._SaveMapFile(MapBuffer, Wa) + self.CreateGuidedSectionToolsFile(Wa) self.invalidateHash() - ## Generate GuidedSectionTools.txt in the FV directories. # - def CreateGuidedSectionToolsFile(self): + def CreateGuidedSectionToolsFile(self,Wa): for BuildTarget in self.BuildTargetList: for ToolChain in self.ToolChainList: - Wa = WorkspaceAutoGen( - self.WorkspaceDir, - self.PlatformFile, - BuildTarget, - ToolChain, - self.ArchList, - self.BuildDatabase, - self.TargetTxt, - self.ToolDef, - self.Fdf, - self.FdList, - self.FvList, - self.CapList, - self.SkuId, - self.UniFlag - ) FvDir = Wa.FvDir if not os.path.exists(FvDir): continue for Arch in self.ArchList: @@ -2285,11 +2373,10 @@ class Build(): if not self.SpawnMode or self.Target not in ["", "all"]: self.SpawnMode = False self._BuildPlatform() else: self._MultiThreadBuildPlatform() - self.CreateGuidedSectionToolsFile() else: self.SpawnMode = False self._BuildModule() if self.Target == 'cleanall': -- 2.20.1.windows.1 ^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen 2019-09-05 9:04 [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen Bob Feng @ 2019-09-05 13:11 ` Liming Gao 2019-09-05 14:04 ` Bob Feng [not found] ` <15C18FCC38905317.10639@groups.io> 0 siblings, 2 replies; 6+ messages in thread From: Liming Gao @ 2019-09-05 13:11 UTC (permalink / raw) To: devel@edk2.groups.io, Feng, Bob C; +Cc: Shi, Steven Bob: Does this change only impact -u option behavior? Thanks Liming > -----Original Message----- > From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of Bob Feng > Sent: Thursday, September 5, 2019 5:05 PM > To: devel@edk2.groups.io > Cc: Gao, Liming <liming.gao@intel.com>; Shi, Steven <steven.shi@intel.com>; Feng, Bob C <bob.c.feng@intel.com> > Subject: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen > > https://bugzilla.tianocore.org/show_bug.cgi?id=2080 > > This patch is to improve build -u option to re-use > GlobalVar_<platformguid>_<arch>.bin file which is > introduced by multiple-process-autogen feature. > > Cc: Liming Gao <liming.gao@intel.com> > Cc: Steven Shi <steven.shi@intel.com> > Signed-off-by: Bob Feng <bob.c.feng@intel.com> > --- > .../Source/Python/AutoGen/AutoGenWorker.py | 2 +- > .../Source/Python/AutoGen/ModuleAutoGen.py | 4 +- > .../Python/AutoGen/ModuleAutoGenHelper.py | 29 ++ > .../Source/Python/AutoGen/PlatformAutoGen.py | 2 + > .../Source/Python/AutoGen/WorkspaceAutoGen.py | 1 - > BaseTools/Source/Python/build/build.py | 355 +++++++++++------- > 6 files changed, 255 insertions(+), 138 deletions(-) > > diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > index 2e68538b1cb4..f488ae9d5f80 100755 > --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process): > Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] > if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: > Ma.GenModuleFilesHash(GlobalData.gCacheIR) > Ma.GenPreMakefileHash(GlobalData.gCacheIR) > if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): > - continue > + continue > > Ma.CreateCodeFile(False) > Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[])) > > if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: > diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py > index 3bb7e91154ac..5f28681e3146 100755 > --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py > +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py > @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen): > not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: > self.GenModuleFilesHash(gDict) > > if not (self.MetaFile.Path, self.Arch) in gDict or \ > not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: > - EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, > self.Arch)) > - return > + EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, > self.Arch)) > + return > > # Initialze hash object > m = hashlib.md5() > > # Add Platform level hash > diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py > index c7591253debd..9dd93b9beb12 100644 > --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py > +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py > @@ -234,22 +234,51 @@ class AutoGenInfo(object): > # > #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in multiple thread > # > class WorkSpaceInfo(AutoGenInfo): > def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch): > + if not hasattr(self, "_Init"): > + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch) > + self._Init = True > + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch): > self._SrcTimeStamp = 0 > self.Db = BuildDB > self.BuildDatabase = self.Db.BuildObject > self.Target = Target > self.ToolChain = ToolChain > self.WorkspaceDir = Workspace > self.ActivePlatform = MetaFile > self.ArchList = Arch > + self.AutoGenObjectList = [] > + @property > + def BuildDir(self): > + return self.AutoGenObjectList[0].BuildDir > > + @property > + def Name(self): > + return self.AutoGenObjectList[0].Platform.PlatformName > + > + @property > + def FlashDefinition(self): > + return self.AutoGenObjectList[0].Platform.FlashDefinition > + @property > + def GenFdsCommandDict(self): > + FdsCommandDict = self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict") > + if FdsCommandDict: > + return FdsCommandDict > + return {} > + > + @cached_property > + def FvDir(self): > + return os.path.join(self.BuildDir, TAB_FV_DIRECTORY) > > class PlatformInfo(AutoGenInfo): > def __init__(self, Workspace, MetaFile, Target, ToolChain, Arch,DataPipe): > + if not hasattr(self, "_Init"): > + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe) > + self._Init = True > + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe): > self.Wa = Workspace > self.WorkspaceDir = self.Wa.WorkspaceDir > self.MetaFile = MetaFile > self.Arch = Arch > self.Target = Target > diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py > index 565424a95ead..1e17b6687129 100644 > --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py > +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py > @@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen): > key = (Ma.MetaFile.File, self.Arch) > if key in FfsCommand: > Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key]) > else: > Ma.CreateMakeFile(CreateModuleMakeFile) > + self.CreateLibModuelDirs() > > + def CreateLibModuelDirs(self): > # no need to create makefile for the platform more than once > if self.IsMakeFileCreated: > return > > # create library/module build dirs for platform > diff --git a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py > index 24942674721f..f9d2c216169c 100644 > --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py > +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py > @@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen): > self._CheckPcdDefineAndType() > > self.CreateBuildOptionsFile() > self.CreatePcdTokenNumberFile() > self.CreateModuleHashInfo() > - GlobalData.gAutoGenPhase = False > > # > # Merge Arch > # > def MergeArch(self): > diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py > index 0406ac314b65..db57c3e526da 100755 > --- a/BaseTools/Source/Python/build/build.py > +++ b/BaseTools/Source/Python/build/build.py > @@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import PatchBinaryFile > > import Common.GlobalData as GlobalData > from GenFds.GenFds import GenFds, GenFdsApi > import multiprocessing as mp > from multiprocessing import Manager > +from AutoGen.DataPipe import MemoryDataPipe > +from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo > +from GenFds.FdfParser import FdfParser > > > ## standard targets of build command > gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run'] > > @@ -837,11 +840,11 @@ class Build(): > for PcdMa in PcdMaList: > if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: > PcdMa.GenModuleFilesHash(share_data) > PcdMa.GenPreMakefileHash(share_data) > if PcdMa.CanSkipbyPreMakefileCache(share_data): > - continue > + continue > > PcdMa.CreateCodeFile(False) > PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, > PcdMa.Arch),[])) > > if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: > @@ -1218,18 +1221,28 @@ class Build(): > mqueue = mp.Queue() > for m in AutoGenObject.GetAllModuleInfo: > mqueue.put(m) > > AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target} > + AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp} > + AutoGenObject.CreateLibModuelDirs() > + AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList} > + AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList} > + AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": AutoGenObject.Workspace.GenFdsCommandDict} > self.Progress.Start("Generating makefile and code") > data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % > (str(AutoGenObject.Guid),AutoGenObject.Arch)) > AutoGenObject.DataPipe.dump(data_pipe_file) > autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, > GlobalData.gCacheIR) > + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") > + with open(AutoGenIdFile,"w") as fw: > + fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList))) > + fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir) > + fw.write("PlatformGuid=%s\n" % str(AutoGenObject.Guid)) > self.Progress.Stop("done!") > if not autogen_rt: > self.AutoGenMgr.TerminateWorkers() > - self.AutoGenMgr.join(0.1) > + self.AutoGenMgr.join(1) > raise FatalError(errorcode) > AutoGenObject.CreateCodeFile(False) > AutoGenObject.CreateMakeFile(False) > else: > # always recreate top/platform makefile when clean, just in case of inconsistency > @@ -1778,10 +1791,11 @@ class Build(): > self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList) > # > # Save MAP buffer into MAP file. > # > self._SaveMapFile (MapBuffer, Wa) > + self.CreateGuidedSectionToolsFile(Wa) > > ## Build active module for different build targets, different tool chains and different archs > # > def _BuildModule(self): > for BuildTarget in self.BuildTargetList: > @@ -1852,13 +1866,13 @@ class Build(): > > if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: > Ma.GenModuleFilesHash(GlobalData.gCacheIR) > Ma.GenPreMakefileHash(GlobalData.gCacheIR) > if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): > - self.HashSkipModules.append(Ma) > - EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) > - continue > + self.HashSkipModules.append(Ma) > + EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) > + continue > > # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds' > if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: > # for target which must generate AutoGen code and makefile > if not self.SkipAutoGen or self.Target == 'genc': > @@ -1987,125 +2001,221 @@ class Build(): > GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData) > for Cmd in GenFfsDict: > tmpInf, tmpArch = GenFfsDict[Cmd] > CmdSetDict[tmpInf, tmpArch].add(Cmd) > return CmdSetDict > + def VerifyAutoGenFiles(self): > + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") > + try: > + with open(AutoGenIdFile) as fd: > + lines = fd.readlines() > + except: > + return None > + for line in lines: > + if "Arch" in line: > + ArchList = line.strip().split("=")[1].split("|") > + if "BuildDir" in line: > + BuildDir = line.split("=")[1].strip() > + if "PlatformGuid" in line: > + PlatformGuid = line.split("=")[1].strip() > + GlobalVarList = [] > + for arch in ArchList: > + global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch)) > + if not os.path.exists(global_var): > + return None > + GlobalVarList.append(global_var) > + for global_var in GlobalVarList: > + data_pipe = MemoryDataPipe() > + data_pipe.load(global_var) > + target = data_pipe.Get("P_Info").get("Target") > + toolchain = data_pipe.Get("P_Info").get("ToolChain") > + archlist = data_pipe.Get("P_Info").get("ArchList") > + Arch = data_pipe.Get("P_Info").get("Arch") > + active_p = data_pipe.Get("P_Info").get("ActivePlatform") > + workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir") > + PackagesPath = os.getenv("PACKAGES_PATH") > + mws.setWs(workspacedir, PackagesPath) > + LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList") > + ModuleBuildDirectoryList = data_pipe.Get("ModuleBuildDirectoryList") > + > + for m_build_dir in LibraryBuildDirectoryList: > + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])): > + return None > + for m_build_dir in ModuleBuildDirectoryList: > + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])): > + return None > + Wa = WorkSpaceInfo( > + workspacedir,active_p,target,toolchain,archlist > + ) > + Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe) > + Wa.AutoGenObjectList.append(Pa) > + return Wa > + def SetupMakeSetting(self,Wa): > + BuildModules = [] > + for Pa in Wa.AutoGenObjectList: > + for m in Pa._MbList: > + ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe) > + BuildModules.append(ma) > + fdf_file = Wa.FlashDefinition > + if fdf_file: > + Fdf = FdfParser(fdf_file.Path) > + Fdf.ParseFile() > + GlobalData.gFdfParser = Fdf > + if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict: > + FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName] > + for FdRegion in FdDict.RegionList: > + if str(FdRegion.RegionType) is 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList): > + if int(FdRegion.Offset) % 8 != 0: > + EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' % > (FdRegion.Offset)) > + Wa.FdfProfile = Fdf.Profile > + self.Fdf = Fdf > + else: > + self.Fdf = None > + return BuildModules > > ## Build a platform in multi-thread mode > # > - def _MultiThreadBuildPlatform(self): > - SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False) > - for BuildTarget in self.BuildTargetList: > - GlobalData.gGlobalDefines['TARGET'] = BuildTarget > - index = 0 > - for ToolChain in self.ToolChainList: > - WorkspaceAutoGenTime = time.time() > - GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain > - GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain > - GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index] > - index += 1 > - Wa = WorkspaceAutoGen( > - self.WorkspaceDir, > - self.PlatformFile, > - BuildTarget, > - ToolChain, > - self.ArchList, > - self.BuildDatabase, > - self.TargetTxt, > - self.ToolDef, > - self.Fdf, > - self.FdList, > - self.FvList, > - self.CapList, > - self.SkuId, > - self.UniFlag, > - self.Progress > - ) > - self.Fdf = Wa.FdfFile > - self.LoadFixAddress = Wa.Platform.LoadFixAddress > - self.BuildReport.AddPlatformReport(Wa) > - Wa.CreateMakeFile(False) > + def PerformAutoGen(self,BuildTarget,ToolChain): > + WorkspaceAutoGenTime = time.time() > + Wa = WorkspaceAutoGen( > + self.WorkspaceDir, > + self.PlatformFile, > + BuildTarget, > + ToolChain, > + self.ArchList, > + self.BuildDatabase, > + self.TargetTxt, > + self.ToolDef, > + self.Fdf, > + self.FdList, > + self.FvList, > + self.CapList, > + self.SkuId, > + self.UniFlag, > + self.Progress > + ) > + self.Fdf = Wa.FdfFile > + self.LoadFixAddress = Wa.Platform.LoadFixAddress > + self.BuildReport.AddPlatformReport(Wa) > + Wa.CreateMakeFile(False) > > # Add ffs build to makefile > - CmdListDict = {} > - if GlobalData.gEnableGenfdsMultiThread and self.Fdf: > - CmdListDict = self._GenFfsCmd(Wa.ArchList) > + CmdListDict = {} > + if GlobalData.gEnableGenfdsMultiThread and self.Fdf: > + CmdListDict = self._GenFfsCmd(Wa.ArchList) > > - # Add Platform and Package level hash in share_data for module hash calculation later > - if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: > - GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash > - for PkgName in GlobalData.gPackageHash.keys(): > - GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName] > + # Add Platform and Package level hash in share_data for module hash calculation later > + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: > + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash > + for PkgName in GlobalData.gPackageHash.keys(): > + GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName] > > - # multi-thread exit flag > - ExitFlag = threading.Event() > - ExitFlag.clear() > - self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) > - self.BuildModules = [] > - TotalModules = [] > - for Arch in Wa.ArchList: > - PcdMaList = [] > - AutoGenStart = time.time() > - GlobalData.gGlobalDefines['ARCH'] = Arch > - Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) > - if Pa is None: > - continue > - ModuleList = [] > - for Inf in Pa.Platform.Modules: > - ModuleList.append(Inf) > + self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) > + BuildModules = [] > + TotalModules = [] > + for Arch in Wa.ArchList: > + PcdMaList = [] > + AutoGenStart = time.time() > + GlobalData.gGlobalDefines['ARCH'] = Arch > + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) > + if Pa is None: > + continue > + ModuleList = [] > + for Inf in Pa.Platform.Modules: > + ModuleList.append(Inf) > # Add the INF only list in FDF > - if GlobalData.gFdfParser is not None: > - for InfName in GlobalData.gFdfParser.Profile.InfList: > - Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch) > - if Inf in Pa.Platform.Modules: > - continue > - ModuleList.append(Inf) > - Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} > - Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} > - Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} > - for Module in ModuleList: > + if GlobalData.gFdfParser is not None: > + for InfName in GlobalData.gFdfParser.Profile.InfList: > + Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch) > + if Inf in Pa.Platform.Modules: > + continue > + ModuleList.append(Inf) > + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} > + Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} > + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} > + Pa.CreateLibModuelDirs() > + Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList} > + Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList} > + Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict} > + ModuleCodaFile = {} > + for ma in Pa.ModuleAutoGenList: > + ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in > ma.CodaTargetList] > + Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile} > + for Module in ModuleList: > # Get ModuleAutoGen object to generate C code file and makefile > - Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) > + Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) > > - if Ma is None: > + if Ma is None: > + continue > + if Ma.PcdIsDriver: > + Ma.PlatformInfo = Pa > + Ma.Workspace = Wa > + PcdMaList.append(Ma) > + TotalModules.append(Ma) > + # Initialize all modules in tracking to 'FAIL' > + GlobalData.gModuleBuildTracking[Ma] = 'FAIL' > + > + > + mqueue = mp.Queue() > + for m in Pa.GetAllModuleInfo: > + mqueue.put(m) > + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) > + Pa.DataPipe.dump(data_pipe_file) > + > + autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR) > + > + # Skip cache hit modules > + if GlobalData.gBinCacheSource: > + for Ma in TotalModules: > + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ > + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit: > + self.HashSkipModules.append(Ma) > + continue > + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ > + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit: > + self.HashSkipModules.append(Ma) > continue > - if Ma.PcdIsDriver: > - Ma.PlatformInfo = Pa > - Ma.Workspace = Wa > - PcdMaList.append(Ma) > - TotalModules.append(Ma) > - # Initialize all modules in tracking to 'FAIL' > - GlobalData.gModuleBuildTracking[Ma] = 'FAIL' > + BuildModules.append(Ma) > + else: > + BuildModules.extend(TotalModules) > > - mqueue = mp.Queue() > - for m in Pa.GetAllModuleInfo: > - mqueue.put(m) > - data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) > - Pa.DataPipe.dump(data_pipe_file) > - autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, > GlobalData.gCacheIR) > + if not autogen_rt: > + self.AutoGenMgr.TerminateWorkers() > + self.AutoGenMgr.join(1) > + raise FatalError(errorcode) > + self.AutoGenTime += int(round((time.time() - AutoGenStart))) > + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") > + with open(AutoGenIdFile,"w") as fw: > + fw.write("Arch=%s\n" % "|".join((Wa.ArchList))) > + fw.write("BuildDir=%s\n" % Wa.BuildDir) > + fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid)) > + self.Progress.Stop("done!") > + return Wa, BuildModules > > - # Skip cache hit modules > - if GlobalData.gBinCacheSource: > - for Ma in TotalModules: > - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ > - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit: > - self.HashSkipModules.append(Ma) > - continue > - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ > - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit: > - self.HashSkipModules.append(Ma) > - continue > - self.BuildModules.append(Ma) > + def _MultiThreadBuildPlatform(self): > + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False) > + for BuildTarget in self.BuildTargetList: > + GlobalData.gGlobalDefines['TARGET'] = BuildTarget > + index = 0 > + for ToolChain in self.ToolChainList: > + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain > + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain > + GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index] > + index += 1 > + ExitFlag = threading.Event() > + ExitFlag.clear() > + if self.SkipAutoGen: > + Wa = self.VerifyAutoGenFiles() > + if Wa is None: > + Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain) > else: > - self.BuildModules.extend(TotalModules) > - > - if not autogen_rt: > - self.AutoGenMgr.TerminateWorkers() > - self.AutoGenMgr.join(0.1) > - raise FatalError(errorcode) > - self.AutoGenTime += int(round((time.time() - AutoGenStart))) > - self.Progress.Stop("done!") > + GlobalData.gAutoGenPhase = True > + self.BuildModules = self.SetupMakeSetting(Wa) > + else: > + Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain) > + Pa = Wa.AutoGenObjectList[0] > + GlobalData.gAutoGenPhase = False > > if GlobalData.gBinCacheSource: > EdkLogger.quiet("Total cache hit driver num: %s, cache miss driver num: %s" % (len(set(self.HashSkipModules)), > len(set(self.BuildModules)))) > CacheHitMa = set() > CacheNotHitMa = set() > @@ -2136,18 +2246,17 @@ class Build(): > # Start task scheduler > if not BuildTask.IsOnGoing(): > BuildTask.StartScheduler(self.ThreadNumber, ExitFlag) > > # in case there's an interruption. we need a full version of makefile for platform > - Pa.CreateMakeFile(False) > + > if BuildTask.HasError(): > self.invalidateHash() > EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) > self.MakeTime += int(round((time.time() - MakeStart))) > > MakeContiue = time.time() > - > # > # > # All modules have been put in build tasks queue. Tell task scheduler > # to exit if all tasks are completed > # > @@ -2175,17 +2284,12 @@ class Build(): > if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and > self.LoadFixAddress >= 0x100000000: > EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to > larger than or equal to 4G for the platorm with IA32 or ARM arch modules") > # > # Get Module List > # > - ModuleList = {} > - for Pa in Wa.AutoGenObjectList: > - for Ma in Pa.ModuleAutoGenList: > - if Ma is None: > - continue > - if not Ma.IsLibrary: > - ModuleList[Ma.Guid.upper()] = Ma > + ModuleList = {ma.Guid.upper():ma for ma in self.BuildModules} > + > # > # Rebase module to the preferred memory address before GenFds > # > MapBuffer = [] > if self.LoadFixAddress != 0: > @@ -2206,33 +2310,17 @@ class Build(): > self.GenFdsTime += int(round((time.time() - GenFdsStart))) > # > # Save MAP buffer into MAP file. > # > self._SaveMapFile(MapBuffer, Wa) > + self.CreateGuidedSectionToolsFile(Wa) > self.invalidateHash() > - > ## Generate GuidedSectionTools.txt in the FV directories. > # > - def CreateGuidedSectionToolsFile(self): > + def CreateGuidedSectionToolsFile(self,Wa): > for BuildTarget in self.BuildTargetList: > for ToolChain in self.ToolChainList: > - Wa = WorkspaceAutoGen( > - self.WorkspaceDir, > - self.PlatformFile, > - BuildTarget, > - ToolChain, > - self.ArchList, > - self.BuildDatabase, > - self.TargetTxt, > - self.ToolDef, > - self.Fdf, > - self.FdList, > - self.FvList, > - self.CapList, > - self.SkuId, > - self.UniFlag > - ) > FvDir = Wa.FvDir > if not os.path.exists(FvDir): > continue > > for Arch in self.ArchList: > @@ -2285,11 +2373,10 @@ class Build(): > if not self.SpawnMode or self.Target not in ["", "all"]: > self.SpawnMode = False > self._BuildPlatform() > else: > self._MultiThreadBuildPlatform() > - self.CreateGuidedSectionToolsFile() > else: > self.SpawnMode = False > self._BuildModule() > > if self.Target == 'cleanall': > -- > 2.20.1.windows.1 > > > ^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen 2019-09-05 13:11 ` [edk2-devel] " Liming Gao @ 2019-09-05 14:04 ` Bob Feng [not found] ` <15C18FCC38905317.10639@groups.io> 1 sibling, 0 replies; 6+ messages in thread From: Bob Feng @ 2019-09-05 14:04 UTC (permalink / raw) To: Gao, Liming, devel@edk2.groups.io; +Cc: Shi, Steven Liming, Yes, this change only impact -u option. I separate _MultiThreadBuildPlatform function into two functions and create 2 new functions specially for IgnoreAutoGen, so the there looks big change for build.py in this patch. I verified this patch on Ovmf, E -Bob -----Original Message----- From: Gao, Liming Sent: Thursday, September 5, 2019 9:12 PM To: devel@edk2.groups.io; Feng, Bob C <bob.c.feng@intel.com> Cc: Shi, Steven <steven.shi@intel.com> Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen Bob: Does this change only impact -u option behavior? Thanks Liming > -----Original Message----- > From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of > Bob Feng > Sent: Thursday, September 5, 2019 5:05 PM > To: devel@edk2.groups.io > Cc: Gao, Liming <liming.gao@intel.com>; Shi, Steven > <steven.shi@intel.com>; Feng, Bob C <bob.c.feng@intel.com> > Subject: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for > IgnoreAutoGen > > https://bugzilla.tianocore.org/show_bug.cgi?id=2080 > > This patch is to improve build -u option to re-use > GlobalVar_<platformguid>_<arch>.bin file which is introduced by > multiple-process-autogen feature. > > Cc: Liming Gao <liming.gao@intel.com> > Cc: Steven Shi <steven.shi@intel.com> > Signed-off-by: Bob Feng <bob.c.feng@intel.com> > --- > .../Source/Python/AutoGen/AutoGenWorker.py | 2 +- > .../Source/Python/AutoGen/ModuleAutoGen.py | 4 +- > .../Python/AutoGen/ModuleAutoGenHelper.py | 29 ++ > .../Source/Python/AutoGen/PlatformAutoGen.py | 2 + > .../Source/Python/AutoGen/WorkspaceAutoGen.py | 1 - > BaseTools/Source/Python/build/build.py | 355 +++++++++++------- > 6 files changed, 255 insertions(+), 138 deletions(-) > > diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > index 2e68538b1cb4..f488ae9d5f80 100755 > --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process): > Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] > if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: > Ma.GenModuleFilesHash(GlobalData.gCacheIR) > Ma.GenPreMakefileHash(GlobalData.gCacheIR) > if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): > - continue > + continue > > Ma.CreateCodeFile(False) > > Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, > Ma.Arch),[])) > > if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: > diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py > b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py > index 3bb7e91154ac..5f28681e3146 100755 > --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py > +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py > @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen): > not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: > self.GenModuleFilesHash(gDict) > > if not (self.MetaFile.Path, self.Arch) in gDict or \ > not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: > - EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, > self.Arch)) > - return > + EdkLogger.quiet("[cache warning]: Cannot generate > + ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, > self.Arch)) > + return > > # Initialze hash object > m = hashlib.md5() > > # Add Platform level hash > diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py > b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py > index c7591253debd..9dd93b9beb12 100644 > --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py > +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py > @@ -234,22 +234,51 @@ class AutoGenInfo(object): > # > #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in > multiple thread # class WorkSpaceInfo(AutoGenInfo): > def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch): > + if not hasattr(self, "_Init"): > + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch) > + self._Init = True > + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch): > self._SrcTimeStamp = 0 > self.Db = BuildDB > self.BuildDatabase = self.Db.BuildObject > self.Target = Target > self.ToolChain = ToolChain > self.WorkspaceDir = Workspace > self.ActivePlatform = MetaFile > self.ArchList = Arch > + self.AutoGenObjectList = [] > + @property > + def BuildDir(self): > + return self.AutoGenObjectList[0].BuildDir > > + @property > + def Name(self): > + return self.AutoGenObjectList[0].Platform.PlatformName > + > + @property > + def FlashDefinition(self): > + return self.AutoGenObjectList[0].Platform.FlashDefinition > + @property > + def GenFdsCommandDict(self): > + FdsCommandDict = self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict") > + if FdsCommandDict: > + return FdsCommandDict > + return {} > + > + @cached_property > + def FvDir(self): > + return os.path.join(self.BuildDir, TAB_FV_DIRECTORY) > > class PlatformInfo(AutoGenInfo): > def __init__(self, Workspace, MetaFile, Target, ToolChain, Arch,DataPipe): > + if not hasattr(self, "_Init"): > + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe) > + self._Init = True > + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe): > self.Wa = Workspace > self.WorkspaceDir = self.Wa.WorkspaceDir > self.MetaFile = MetaFile > self.Arch = Arch > self.Target = Target > diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py > b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py > index 565424a95ead..1e17b6687129 100644 > --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py > +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py > @@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen): > key = (Ma.MetaFile.File, self.Arch) > if key in FfsCommand: > Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key]) > else: > Ma.CreateMakeFile(CreateModuleMakeFile) > + self.CreateLibModuelDirs() > > + def CreateLibModuelDirs(self): > # no need to create makefile for the platform more than once > if self.IsMakeFileCreated: > return > > # create library/module build dirs for platform diff --git > a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py > b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py > index 24942674721f..f9d2c216169c 100644 > --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py > +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py > @@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen): > self._CheckPcdDefineAndType() > > self.CreateBuildOptionsFile() > self.CreatePcdTokenNumberFile() > self.CreateModuleHashInfo() > - GlobalData.gAutoGenPhase = False > > # > # Merge Arch > # > def MergeArch(self): > diff --git a/BaseTools/Source/Python/build/build.py > b/BaseTools/Source/Python/build/build.py > index 0406ac314b65..db57c3e526da 100755 > --- a/BaseTools/Source/Python/build/build.py > +++ b/BaseTools/Source/Python/build/build.py > @@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import > PatchBinaryFile > > import Common.GlobalData as GlobalData from GenFds.GenFds import > GenFds, GenFdsApi import multiprocessing as mp from multiprocessing > import Manager > +from AutoGen.DataPipe import MemoryDataPipe from > +AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo from > +GenFds.FdfParser import FdfParser > > > ## standard targets of build command > gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', > 'fds', 'clean', 'cleanall', 'cleanlib', 'run'] > > @@ -837,11 +840,11 @@ class Build(): > for PcdMa in PcdMaList: > if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: > PcdMa.GenModuleFilesHash(share_data) > PcdMa.GenPreMakefileHash(share_data) > if PcdMa.CanSkipbyPreMakefileCache(share_data): > - continue > + continue > > PcdMa.CreateCodeFile(False) > PcdMa.CreateMakeFile(False,GenFfsList = > DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, > PcdMa.Arch),[])) > > if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: > @@ -1218,18 +1221,28 @@ class Build(): > mqueue = mp.Queue() > for m in AutoGenObject.GetAllModuleInfo: > mqueue.put(m) > > AutoGenObject.DataPipe.DataContainer = {"CommandTarget": > self.Target} > + AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp} > + AutoGenObject.CreateLibModuelDirs() > + AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList} > + AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList} > + AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": > + AutoGenObject.Workspace.GenFdsCommandDict} > self.Progress.Start("Generating makefile and code") > data_pipe_file = os.path.join(AutoGenObject.BuildDir, > "GlobalVar_%s_%s.bin" % > (str(AutoGenObject.Guid),AutoGenObject.Arch)) > AutoGenObject.DataPipe.dump(data_pipe_file) > autogen_rt,errorcode = self.StartAutoGen(mqueue, > AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, > GlobalData.gCacheIR) > + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") > + with open(AutoGenIdFile,"w") as fw: > + fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList))) > + fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir) > + fw.write("PlatformGuid=%s\n" % > + str(AutoGenObject.Guid)) > self.Progress.Stop("done!") > if not autogen_rt: > self.AutoGenMgr.TerminateWorkers() > - self.AutoGenMgr.join(0.1) > + self.AutoGenMgr.join(1) > raise FatalError(errorcode) > AutoGenObject.CreateCodeFile(False) > AutoGenObject.CreateMakeFile(False) > else: > # always recreate top/platform makefile when clean, just > in case of inconsistency @@ -1778,10 +1791,11 @@ class Build(): > self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList) > # > # Save MAP buffer into MAP file. > # > self._SaveMapFile (MapBuffer, Wa) > + self.CreateGuidedSectionToolsFile(Wa) > > ## Build active module for different build targets, different tool chains and different archs > # > def _BuildModule(self): > for BuildTarget in self.BuildTargetList: > @@ -1852,13 +1866,13 @@ class Build(): > > if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: > Ma.GenModuleFilesHash(GlobalData.gCacheIR) > Ma.GenPreMakefileHash(GlobalData.gCacheIR) > if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): > - self.HashSkipModules.append(Ma) > - EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) > - continue > + self.HashSkipModules.append(Ma) > + EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) > + continue > > # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds' > if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: > # for target which must generate AutoGen code and makefile > if not self.SkipAutoGen or self.Target == 'genc': > @@ -1987,125 +2001,221 @@ class Build(): > GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData) > for Cmd in GenFfsDict: > tmpInf, tmpArch = GenFfsDict[Cmd] > CmdSetDict[tmpInf, tmpArch].add(Cmd) > return CmdSetDict > + def VerifyAutoGenFiles(self): > + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") > + try: > + with open(AutoGenIdFile) as fd: > + lines = fd.readlines() > + except: > + return None > + for line in lines: > + if "Arch" in line: > + ArchList = line.strip().split("=")[1].split("|") > + if "BuildDir" in line: > + BuildDir = line.split("=")[1].strip() > + if "PlatformGuid" in line: > + PlatformGuid = line.split("=")[1].strip() > + GlobalVarList = [] > + for arch in ArchList: > + global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch)) > + if not os.path.exists(global_var): > + return None > + GlobalVarList.append(global_var) > + for global_var in GlobalVarList: > + data_pipe = MemoryDataPipe() > + data_pipe.load(global_var) > + target = data_pipe.Get("P_Info").get("Target") > + toolchain = data_pipe.Get("P_Info").get("ToolChain") > + archlist = data_pipe.Get("P_Info").get("ArchList") > + Arch = data_pipe.Get("P_Info").get("Arch") > + active_p = data_pipe.Get("P_Info").get("ActivePlatform") > + workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir") > + PackagesPath = os.getenv("PACKAGES_PATH") > + mws.setWs(workspacedir, PackagesPath) > + LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList") > + ModuleBuildDirectoryList = > + data_pipe.Get("ModuleBuildDirectoryList") > + > + for m_build_dir in LibraryBuildDirectoryList: > + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])): > + return None > + for m_build_dir in ModuleBuildDirectoryList: > + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])): > + return None > + Wa = WorkSpaceInfo( > + workspacedir,active_p,target,toolchain,archlist > + ) > + Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe) > + Wa.AutoGenObjectList.append(Pa) > + return Wa > + def SetupMakeSetting(self,Wa): > + BuildModules = [] > + for Pa in Wa.AutoGenObjectList: > + for m in Pa._MbList: > + ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe) > + BuildModules.append(ma) > + fdf_file = Wa.FlashDefinition > + if fdf_file: > + Fdf = FdfParser(fdf_file.Path) > + Fdf.ParseFile() > + GlobalData.gFdfParser = Fdf > + if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict: > + FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName] > + for FdRegion in FdDict.RegionList: > + if str(FdRegion.RegionType) is 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList): > + if int(FdRegion.Offset) % 8 != 0: > + EdkLogger.error("build", FORMAT_INVALID, > + 'The VPD Base Address %s must be 8-byte aligned.' % > (FdRegion.Offset)) > + Wa.FdfProfile = Fdf.Profile > + self.Fdf = Fdf > + else: > + self.Fdf = None > + return BuildModules > > ## Build a platform in multi-thread mode > # > - def _MultiThreadBuildPlatform(self): > - SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False) > - for BuildTarget in self.BuildTargetList: > - GlobalData.gGlobalDefines['TARGET'] = BuildTarget > - index = 0 > - for ToolChain in self.ToolChainList: > - WorkspaceAutoGenTime = time.time() > - GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain > - GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain > - GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index] > - index += 1 > - Wa = WorkspaceAutoGen( > - self.WorkspaceDir, > - self.PlatformFile, > - BuildTarget, > - ToolChain, > - self.ArchList, > - self.BuildDatabase, > - self.TargetTxt, > - self.ToolDef, > - self.Fdf, > - self.FdList, > - self.FvList, > - self.CapList, > - self.SkuId, > - self.UniFlag, > - self.Progress > - ) > - self.Fdf = Wa.FdfFile > - self.LoadFixAddress = Wa.Platform.LoadFixAddress > - self.BuildReport.AddPlatformReport(Wa) > - Wa.CreateMakeFile(False) > + def PerformAutoGen(self,BuildTarget,ToolChain): > + WorkspaceAutoGenTime = time.time() > + Wa = WorkspaceAutoGen( > + self.WorkspaceDir, > + self.PlatformFile, > + BuildTarget, > + ToolChain, > + self.ArchList, > + self.BuildDatabase, > + self.TargetTxt, > + self.ToolDef, > + self.Fdf, > + self.FdList, > + self.FvList, > + self.CapList, > + self.SkuId, > + self.UniFlag, > + self.Progress > + ) > + self.Fdf = Wa.FdfFile > + self.LoadFixAddress = Wa.Platform.LoadFixAddress > + self.BuildReport.AddPlatformReport(Wa) > + Wa.CreateMakeFile(False) > > # Add ffs build to makefile > - CmdListDict = {} > - if GlobalData.gEnableGenfdsMultiThread and self.Fdf: > - CmdListDict = self._GenFfsCmd(Wa.ArchList) > + CmdListDict = {} > + if GlobalData.gEnableGenfdsMultiThread and self.Fdf: > + CmdListDict = self._GenFfsCmd(Wa.ArchList) > > - # Add Platform and Package level hash in share_data for module hash calculation later > - if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: > - GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash > - for PkgName in GlobalData.gPackageHash.keys(): > - GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName] > + # Add Platform and Package level hash in share_data for module hash calculation later > + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: > + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash > + for PkgName in GlobalData.gPackageHash.keys(): > + GlobalData.gCacheIR[(PkgName, 'PackageHash')] = > + GlobalData.gPackageHash[PkgName] > > - # multi-thread exit flag > - ExitFlag = threading.Event() > - ExitFlag.clear() > - self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) > - self.BuildModules = [] > - TotalModules = [] > - for Arch in Wa.ArchList: > - PcdMaList = [] > - AutoGenStart = time.time() > - GlobalData.gGlobalDefines['ARCH'] = Arch > - Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) > - if Pa is None: > - continue > - ModuleList = [] > - for Inf in Pa.Platform.Modules: > - ModuleList.append(Inf) > + self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) > + BuildModules = [] > + TotalModules = [] > + for Arch in Wa.ArchList: > + PcdMaList = [] > + AutoGenStart = time.time() > + GlobalData.gGlobalDefines['ARCH'] = Arch > + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) > + if Pa is None: > + continue > + ModuleList = [] > + for Inf in Pa.Platform.Modules: > + ModuleList.append(Inf) > # Add the INF only list in FDF > - if GlobalData.gFdfParser is not None: > - for InfName in GlobalData.gFdfParser.Profile.InfList: > - Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch) > - if Inf in Pa.Platform.Modules: > - continue > - ModuleList.append(Inf) > - Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} > - Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} > - Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} > - for Module in ModuleList: > + if GlobalData.gFdfParser is not None: > + for InfName in GlobalData.gFdfParser.Profile.InfList: > + Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch) > + if Inf in Pa.Platform.Modules: > + continue > + ModuleList.append(Inf) > + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} > + Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} > + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} > + Pa.CreateLibModuelDirs() > + Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList} > + Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList} > + Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict} > + ModuleCodaFile = {} > + for ma in Pa.ModuleAutoGenList: > + > + ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFil > + e.Path)] = [item.Target for item in > ma.CodaTargetList] > + Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile} > + for Module in ModuleList: > # Get ModuleAutoGen object to generate C code file and makefile > - Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) > + Ma = ModuleAutoGen(Wa, Module, BuildTarget, > + ToolChain, Arch, self.PlatformFile,Pa.DataPipe) > > - if Ma is None: > + if Ma is None: > + continue > + if Ma.PcdIsDriver: > + Ma.PlatformInfo = Pa > + Ma.Workspace = Wa > + PcdMaList.append(Ma) > + TotalModules.append(Ma) > + # Initialize all modules in tracking to 'FAIL' > + GlobalData.gModuleBuildTracking[Ma] = 'FAIL' > + > + > + mqueue = mp.Queue() > + for m in Pa.GetAllModuleInfo: > + mqueue.put(m) > + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) > + Pa.DataPipe.dump(data_pipe_file) > + > + autogen_rt, errorcode = self.StartAutoGen(mqueue, > + Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR) > + > + # Skip cache hit modules > + if GlobalData.gBinCacheSource: > + for Ma in TotalModules: > + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ > + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit: > + self.HashSkipModules.append(Ma) > + continue > + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ > + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit: > + self.HashSkipModules.append(Ma) > continue > - if Ma.PcdIsDriver: > - Ma.PlatformInfo = Pa > - Ma.Workspace = Wa > - PcdMaList.append(Ma) > - TotalModules.append(Ma) > - # Initialize all modules in tracking to 'FAIL' > - GlobalData.gModuleBuildTracking[Ma] = 'FAIL' > + BuildModules.append(Ma) > + else: > + BuildModules.extend(TotalModules) > > - mqueue = mp.Queue() > - for m in Pa.GetAllModuleInfo: > - mqueue.put(m) > - data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) > - Pa.DataPipe.dump(data_pipe_file) > - autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, > GlobalData.gCacheIR) > + if not autogen_rt: > + self.AutoGenMgr.TerminateWorkers() > + self.AutoGenMgr.join(1) > + raise FatalError(errorcode) > + self.AutoGenTime += int(round((time.time() - AutoGenStart))) > + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") > + with open(AutoGenIdFile,"w") as fw: > + fw.write("Arch=%s\n" % "|".join((Wa.ArchList))) > + fw.write("BuildDir=%s\n" % Wa.BuildDir) > + fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid)) > + self.Progress.Stop("done!") > + return Wa, BuildModules > > - # Skip cache hit modules > - if GlobalData.gBinCacheSource: > - for Ma in TotalModules: > - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ > - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit: > - self.HashSkipModules.append(Ma) > - continue > - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ > - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit: > - self.HashSkipModules.append(Ma) > - continue > - self.BuildModules.append(Ma) > + def _MultiThreadBuildPlatform(self): > + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False) > + for BuildTarget in self.BuildTargetList: > + GlobalData.gGlobalDefines['TARGET'] = BuildTarget > + index = 0 > + for ToolChain in self.ToolChainList: > + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain > + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain > + GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index] > + index += 1 > + ExitFlag = threading.Event() > + ExitFlag.clear() > + if self.SkipAutoGen: > + Wa = self.VerifyAutoGenFiles() > + if Wa is None: > + Wa, self.BuildModules = > + self.PerformAutoGen(BuildTarget,ToolChain) > else: > - self.BuildModules.extend(TotalModules) > - > - if not autogen_rt: > - self.AutoGenMgr.TerminateWorkers() > - self.AutoGenMgr.join(0.1) > - raise FatalError(errorcode) > - self.AutoGenTime += int(round((time.time() - AutoGenStart))) > - self.Progress.Stop("done!") > + GlobalData.gAutoGenPhase = True > + self.BuildModules = self.SetupMakeSetting(Wa) > + else: > + Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain) > + Pa = Wa.AutoGenObjectList[0] > + GlobalData.gAutoGenPhase = False > > if GlobalData.gBinCacheSource: > EdkLogger.quiet("Total cache hit driver num: %s, > cache miss driver num: %s" % (len(set(self.HashSkipModules)), > len(set(self.BuildModules)))) > CacheHitMa = set() > CacheNotHitMa = set() @@ -2136,18 +2246,17 @@ > class Build(): > # Start task scheduler > if not BuildTask.IsOnGoing(): > > BuildTask.StartScheduler(self.ThreadNumber, ExitFlag) > > # in case there's an interruption. we need a full version of makefile for platform > - Pa.CreateMakeFile(False) > + > if BuildTask.HasError(): > self.invalidateHash() > EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) > self.MakeTime += int(round((time.time() - > MakeStart))) > > MakeContiue = time.time() > - > # > # > # All modules have been put in build tasks queue. Tell task scheduler > # to exit if all tasks are completed > # > @@ -2175,17 +2284,12 @@ class Build(): > if (Arch == 'IA32' or Arch == 'ARM') and > self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000: > EdkLogger.error("build", > PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules") > # > # Get Module List > # > - ModuleList = {} > - for Pa in Wa.AutoGenObjectList: > - for Ma in Pa.ModuleAutoGenList: > - if Ma is None: > - continue > - if not Ma.IsLibrary: > - ModuleList[Ma.Guid.upper()] = Ma > + ModuleList = {ma.Guid.upper():ma for ma in > + self.BuildModules} > + > # > # Rebase module to the preferred memory address before GenFds > # > MapBuffer = [] > if self.LoadFixAddress != 0: > @@ -2206,33 +2310,17 @@ class Build(): > self.GenFdsTime += int(round((time.time() - GenFdsStart))) > # > # Save MAP buffer into MAP file. > # > self._SaveMapFile(MapBuffer, Wa) > + self.CreateGuidedSectionToolsFile(Wa) > self.invalidateHash() > - > ## Generate GuidedSectionTools.txt in the FV directories. > # > - def CreateGuidedSectionToolsFile(self): > + def CreateGuidedSectionToolsFile(self,Wa): > for BuildTarget in self.BuildTargetList: > for ToolChain in self.ToolChainList: > - Wa = WorkspaceAutoGen( > - self.WorkspaceDir, > - self.PlatformFile, > - BuildTarget, > - ToolChain, > - self.ArchList, > - self.BuildDatabase, > - self.TargetTxt, > - self.ToolDef, > - self.Fdf, > - self.FdList, > - self.FvList, > - self.CapList, > - self.SkuId, > - self.UniFlag > - ) > FvDir = Wa.FvDir > if not os.path.exists(FvDir): > continue > > for Arch in self.ArchList: > @@ -2285,11 +2373,10 @@ class Build(): > if not self.SpawnMode or self.Target not in ["", "all"]: > self.SpawnMode = False > self._BuildPlatform() > else: > self._MultiThreadBuildPlatform() > - self.CreateGuidedSectionToolsFile() > else: > self.SpawnMode = False > self._BuildModule() > > if self.Target == 'cleanall': > -- > 2.20.1.windows.1 > > > ^ permalink raw reply [flat|nested] 6+ messages in thread
[parent not found: <15C18FCC38905317.10639@groups.io>]
* Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen [not found] ` <15C18FCC38905317.10639@groups.io> @ 2019-09-05 14:05 ` Bob Feng 2019-09-09 6:58 ` Liming Gao [not found] ` <15C2B2E9C55F0B32.16248@groups.io> 0 siblings, 2 replies; 6+ messages in thread From: Bob Feng @ 2019-09-05 14:05 UTC (permalink / raw) To: devel@edk2.groups.io, Feng, Bob C, Gao, Liming; +Cc: Shi, Steven Sorry for the incomplete mail. I verified this patch on Ovmf, Emulator, MinPlatforms and Intel Server Platform. Thanks, Bob -----Original Message----- From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Bob Feng Sent: Thursday, September 5, 2019 10:04 PM To: Gao, Liming <liming.gao@intel.com>; devel@edk2.groups.io Cc: Shi, Steven <steven.shi@intel.com> Subject: Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen Liming, Yes, this change only impact -u option. I separate _MultiThreadBuildPlatform function into two functions and create 2 new functions specially for IgnoreAutoGen, so the there looks big change for build.py in this patch. I verified this patch on Ovmf, E -Bob -----Original Message----- From: Gao, Liming Sent: Thursday, September 5, 2019 9:12 PM To: devel@edk2.groups.io; Feng, Bob C <bob.c.feng@intel.com> Cc: Shi, Steven <steven.shi@intel.com> Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen Bob: Does this change only impact -u option behavior? Thanks Liming > -----Original Message----- > From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of > Bob Feng > Sent: Thursday, September 5, 2019 5:05 PM > To: devel@edk2.groups.io > Cc: Gao, Liming <liming.gao@intel.com>; Shi, Steven > <steven.shi@intel.com>; Feng, Bob C <bob.c.feng@intel.com> > Subject: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for > IgnoreAutoGen > > https://bugzilla.tianocore.org/show_bug.cgi?id=2080 > > This patch is to improve build -u option to re-use > GlobalVar_<platformguid>_<arch>.bin file which is introduced by > multiple-process-autogen feature. > > Cc: Liming Gao <liming.gao@intel.com> > Cc: Steven Shi <steven.shi@intel.com> > Signed-off-by: Bob Feng <bob.c.feng@intel.com> > --- > .../Source/Python/AutoGen/AutoGenWorker.py | 2 +- > .../Source/Python/AutoGen/ModuleAutoGen.py | 4 +- > .../Python/AutoGen/ModuleAutoGenHelper.py | 29 ++ > .../Source/Python/AutoGen/PlatformAutoGen.py | 2 + > .../Source/Python/AutoGen/WorkspaceAutoGen.py | 1 - > BaseTools/Source/Python/build/build.py | 355 +++++++++++------- > 6 files changed, 255 insertions(+), 138 deletions(-) > > diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > index 2e68538b1cb4..f488ae9d5f80 100755 > --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process): > Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] > if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: > Ma.GenModuleFilesHash(GlobalData.gCacheIR) > Ma.GenPreMakefileHash(GlobalData.gCacheIR) > if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): > - continue > + continue > > Ma.CreateCodeFile(False) > > Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, > Ma.Arch),[])) > > if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: > diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py > b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py > index 3bb7e91154ac..5f28681e3146 100755 > --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py > +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py > @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen): > not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: > self.GenModuleFilesHash(gDict) > > if not (self.MetaFile.Path, self.Arch) in gDict or \ > not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: > - EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, > self.Arch)) > - return > + EdkLogger.quiet("[cache warning]: Cannot generate > + ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, > self.Arch)) > + return > > # Initialze hash object > m = hashlib.md5() > > # Add Platform level hash > diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py > b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py > index c7591253debd..9dd93b9beb12 100644 > --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py > +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py > @@ -234,22 +234,51 @@ class AutoGenInfo(object): > # > #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in > multiple thread # class WorkSpaceInfo(AutoGenInfo): > def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch): > + if not hasattr(self, "_Init"): > + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch) > + self._Init = True > + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch): > self._SrcTimeStamp = 0 > self.Db = BuildDB > self.BuildDatabase = self.Db.BuildObject > self.Target = Target > self.ToolChain = ToolChain > self.WorkspaceDir = Workspace > self.ActivePlatform = MetaFile > self.ArchList = Arch > + self.AutoGenObjectList = [] > + @property > + def BuildDir(self): > + return self.AutoGenObjectList[0].BuildDir > > + @property > + def Name(self): > + return self.AutoGenObjectList[0].Platform.PlatformName > + > + @property > + def FlashDefinition(self): > + return self.AutoGenObjectList[0].Platform.FlashDefinition > + @property > + def GenFdsCommandDict(self): > + FdsCommandDict = self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict") > + if FdsCommandDict: > + return FdsCommandDict > + return {} > + > + @cached_property > + def FvDir(self): > + return os.path.join(self.BuildDir, TAB_FV_DIRECTORY) > > class PlatformInfo(AutoGenInfo): > def __init__(self, Workspace, MetaFile, Target, ToolChain, Arch,DataPipe): > + if not hasattr(self, "_Init"): > + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe) > + self._Init = True > + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe): > self.Wa = Workspace > self.WorkspaceDir = self.Wa.WorkspaceDir > self.MetaFile = MetaFile > self.Arch = Arch > self.Target = Target > diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py > b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py > index 565424a95ead..1e17b6687129 100644 > --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py > +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py > @@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen): > key = (Ma.MetaFile.File, self.Arch) > if key in FfsCommand: > Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key]) > else: > Ma.CreateMakeFile(CreateModuleMakeFile) > + self.CreateLibModuelDirs() > > + def CreateLibModuelDirs(self): > # no need to create makefile for the platform more than once > if self.IsMakeFileCreated: > return > > # create library/module build dirs for platform diff --git > a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py > b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py > index 24942674721f..f9d2c216169c 100644 > --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py > +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py > @@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen): > self._CheckPcdDefineAndType() > > self.CreateBuildOptionsFile() > self.CreatePcdTokenNumberFile() > self.CreateModuleHashInfo() > - GlobalData.gAutoGenPhase = False > > # > # Merge Arch > # > def MergeArch(self): > diff --git a/BaseTools/Source/Python/build/build.py > b/BaseTools/Source/Python/build/build.py > index 0406ac314b65..db57c3e526da 100755 > --- a/BaseTools/Source/Python/build/build.py > +++ b/BaseTools/Source/Python/build/build.py > @@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import > PatchBinaryFile > > import Common.GlobalData as GlobalData from GenFds.GenFds import > GenFds, GenFdsApi import multiprocessing as mp from multiprocessing > import Manager > +from AutoGen.DataPipe import MemoryDataPipe from > +AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo from > +GenFds.FdfParser import FdfParser > > > ## standard targets of build command > gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', > 'fds', 'clean', 'cleanall', 'cleanlib', 'run'] > > @@ -837,11 +840,11 @@ class Build(): > for PcdMa in PcdMaList: > if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: > PcdMa.GenModuleFilesHash(share_data) > PcdMa.GenPreMakefileHash(share_data) > if PcdMa.CanSkipbyPreMakefileCache(share_data): > - continue > + continue > > PcdMa.CreateCodeFile(False) > PcdMa.CreateMakeFile(False,GenFfsList = > DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, > PcdMa.Arch),[])) > > if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: > @@ -1218,18 +1221,28 @@ class Build(): > mqueue = mp.Queue() > for m in AutoGenObject.GetAllModuleInfo: > mqueue.put(m) > > AutoGenObject.DataPipe.DataContainer = {"CommandTarget": > self.Target} > + AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp} > + AutoGenObject.CreateLibModuelDirs() > + AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList} > + AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList} > + AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": > + AutoGenObject.Workspace.GenFdsCommandDict} > self.Progress.Start("Generating makefile and code") > data_pipe_file = os.path.join(AutoGenObject.BuildDir, > "GlobalVar_%s_%s.bin" % > (str(AutoGenObject.Guid),AutoGenObject.Arch)) > AutoGenObject.DataPipe.dump(data_pipe_file) > autogen_rt,errorcode = self.StartAutoGen(mqueue, > AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, > GlobalData.gCacheIR) > + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") > + with open(AutoGenIdFile,"w") as fw: > + fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList))) > + fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir) > + fw.write("PlatformGuid=%s\n" % > + str(AutoGenObject.Guid)) > self.Progress.Stop("done!") > if not autogen_rt: > self.AutoGenMgr.TerminateWorkers() > - self.AutoGenMgr.join(0.1) > + self.AutoGenMgr.join(1) > raise FatalError(errorcode) > AutoGenObject.CreateCodeFile(False) > AutoGenObject.CreateMakeFile(False) > else: > # always recreate top/platform makefile when clean, just > in case of inconsistency @@ -1778,10 +1791,11 @@ class Build(): > self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList) > # > # Save MAP buffer into MAP file. > # > self._SaveMapFile (MapBuffer, Wa) > + self.CreateGuidedSectionToolsFile(Wa) > > ## Build active module for different build targets, different tool chains and different archs > # > def _BuildModule(self): > for BuildTarget in self.BuildTargetList: > @@ -1852,13 +1866,13 @@ class Build(): > > if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: > Ma.GenModuleFilesHash(GlobalData.gCacheIR) > Ma.GenPreMakefileHash(GlobalData.gCacheIR) > if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): > - self.HashSkipModules.append(Ma) > - EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) > - continue > + self.HashSkipModules.append(Ma) > + EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) > + continue > > # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds' > if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: > # for target which must generate AutoGen code and makefile > if not self.SkipAutoGen or self.Target == 'genc': > @@ -1987,125 +2001,221 @@ class Build(): > GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData) > for Cmd in GenFfsDict: > tmpInf, tmpArch = GenFfsDict[Cmd] > CmdSetDict[tmpInf, tmpArch].add(Cmd) > return CmdSetDict > + def VerifyAutoGenFiles(self): > + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") > + try: > + with open(AutoGenIdFile) as fd: > + lines = fd.readlines() > + except: > + return None > + for line in lines: > + if "Arch" in line: > + ArchList = line.strip().split("=")[1].split("|") > + if "BuildDir" in line: > + BuildDir = line.split("=")[1].strip() > + if "PlatformGuid" in line: > + PlatformGuid = line.split("=")[1].strip() > + GlobalVarList = [] > + for arch in ArchList: > + global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch)) > + if not os.path.exists(global_var): > + return None > + GlobalVarList.append(global_var) > + for global_var in GlobalVarList: > + data_pipe = MemoryDataPipe() > + data_pipe.load(global_var) > + target = data_pipe.Get("P_Info").get("Target") > + toolchain = data_pipe.Get("P_Info").get("ToolChain") > + archlist = data_pipe.Get("P_Info").get("ArchList") > + Arch = data_pipe.Get("P_Info").get("Arch") > + active_p = data_pipe.Get("P_Info").get("ActivePlatform") > + workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir") > + PackagesPath = os.getenv("PACKAGES_PATH") > + mws.setWs(workspacedir, PackagesPath) > + LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList") > + ModuleBuildDirectoryList = > + data_pipe.Get("ModuleBuildDirectoryList") > + > + for m_build_dir in LibraryBuildDirectoryList: > + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])): > + return None > + for m_build_dir in ModuleBuildDirectoryList: > + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])): > + return None > + Wa = WorkSpaceInfo( > + workspacedir,active_p,target,toolchain,archlist > + ) > + Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe) > + Wa.AutoGenObjectList.append(Pa) > + return Wa > + def SetupMakeSetting(self,Wa): > + BuildModules = [] > + for Pa in Wa.AutoGenObjectList: > + for m in Pa._MbList: > + ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe) > + BuildModules.append(ma) > + fdf_file = Wa.FlashDefinition > + if fdf_file: > + Fdf = FdfParser(fdf_file.Path) > + Fdf.ParseFile() > + GlobalData.gFdfParser = Fdf > + if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict: > + FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName] > + for FdRegion in FdDict.RegionList: > + if str(FdRegion.RegionType) is 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList): > + if int(FdRegion.Offset) % 8 != 0: > + EdkLogger.error("build", FORMAT_INVALID, > + 'The VPD Base Address %s must be 8-byte aligned.' % > (FdRegion.Offset)) > + Wa.FdfProfile = Fdf.Profile > + self.Fdf = Fdf > + else: > + self.Fdf = None > + return BuildModules > > ## Build a platform in multi-thread mode > # > - def _MultiThreadBuildPlatform(self): > - SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False) > - for BuildTarget in self.BuildTargetList: > - GlobalData.gGlobalDefines['TARGET'] = BuildTarget > - index = 0 > - for ToolChain in self.ToolChainList: > - WorkspaceAutoGenTime = time.time() > - GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain > - GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain > - GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index] > - index += 1 > - Wa = WorkspaceAutoGen( > - self.WorkspaceDir, > - self.PlatformFile, > - BuildTarget, > - ToolChain, > - self.ArchList, > - self.BuildDatabase, > - self.TargetTxt, > - self.ToolDef, > - self.Fdf, > - self.FdList, > - self.FvList, > - self.CapList, > - self.SkuId, > - self.UniFlag, > - self.Progress > - ) > - self.Fdf = Wa.FdfFile > - self.LoadFixAddress = Wa.Platform.LoadFixAddress > - self.BuildReport.AddPlatformReport(Wa) > - Wa.CreateMakeFile(False) > + def PerformAutoGen(self,BuildTarget,ToolChain): > + WorkspaceAutoGenTime = time.time() > + Wa = WorkspaceAutoGen( > + self.WorkspaceDir, > + self.PlatformFile, > + BuildTarget, > + ToolChain, > + self.ArchList, > + self.BuildDatabase, > + self.TargetTxt, > + self.ToolDef, > + self.Fdf, > + self.FdList, > + self.FvList, > + self.CapList, > + self.SkuId, > + self.UniFlag, > + self.Progress > + ) > + self.Fdf = Wa.FdfFile > + self.LoadFixAddress = Wa.Platform.LoadFixAddress > + self.BuildReport.AddPlatformReport(Wa) > + Wa.CreateMakeFile(False) > > # Add ffs build to makefile > - CmdListDict = {} > - if GlobalData.gEnableGenfdsMultiThread and self.Fdf: > - CmdListDict = self._GenFfsCmd(Wa.ArchList) > + CmdListDict = {} > + if GlobalData.gEnableGenfdsMultiThread and self.Fdf: > + CmdListDict = self._GenFfsCmd(Wa.ArchList) > > - # Add Platform and Package level hash in share_data for module hash calculation later > - if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: > - GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash > - for PkgName in GlobalData.gPackageHash.keys(): > - GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName] > + # Add Platform and Package level hash in share_data for module hash calculation later > + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: > + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash > + for PkgName in GlobalData.gPackageHash.keys(): > + GlobalData.gCacheIR[(PkgName, 'PackageHash')] = > + GlobalData.gPackageHash[PkgName] > > - # multi-thread exit flag > - ExitFlag = threading.Event() > - ExitFlag.clear() > - self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) > - self.BuildModules = [] > - TotalModules = [] > - for Arch in Wa.ArchList: > - PcdMaList = [] > - AutoGenStart = time.time() > - GlobalData.gGlobalDefines['ARCH'] = Arch > - Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) > - if Pa is None: > - continue > - ModuleList = [] > - for Inf in Pa.Platform.Modules: > - ModuleList.append(Inf) > + self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) > + BuildModules = [] > + TotalModules = [] > + for Arch in Wa.ArchList: > + PcdMaList = [] > + AutoGenStart = time.time() > + GlobalData.gGlobalDefines['ARCH'] = Arch > + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) > + if Pa is None: > + continue > + ModuleList = [] > + for Inf in Pa.Platform.Modules: > + ModuleList.append(Inf) > # Add the INF only list in FDF > - if GlobalData.gFdfParser is not None: > - for InfName in GlobalData.gFdfParser.Profile.InfList: > - Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch) > - if Inf in Pa.Platform.Modules: > - continue > - ModuleList.append(Inf) > - Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} > - Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} > - Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} > - for Module in ModuleList: > + if GlobalData.gFdfParser is not None: > + for InfName in GlobalData.gFdfParser.Profile.InfList: > + Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch) > + if Inf in Pa.Platform.Modules: > + continue > + ModuleList.append(Inf) > + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} > + Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} > + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} > + Pa.CreateLibModuelDirs() > + Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList} > + Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList} > + Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict} > + ModuleCodaFile = {} > + for ma in Pa.ModuleAutoGenList: > + > + ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFil > + e.Path)] = [item.Target for item in > ma.CodaTargetList] > + Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile} > + for Module in ModuleList: > # Get ModuleAutoGen object to generate C code file and makefile > - Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) > + Ma = ModuleAutoGen(Wa, Module, BuildTarget, > + ToolChain, Arch, self.PlatformFile,Pa.DataPipe) > > - if Ma is None: > + if Ma is None: > + continue > + if Ma.PcdIsDriver: > + Ma.PlatformInfo = Pa > + Ma.Workspace = Wa > + PcdMaList.append(Ma) > + TotalModules.append(Ma) > + # Initialize all modules in tracking to 'FAIL' > + GlobalData.gModuleBuildTracking[Ma] = 'FAIL' > + > + > + mqueue = mp.Queue() > + for m in Pa.GetAllModuleInfo: > + mqueue.put(m) > + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) > + Pa.DataPipe.dump(data_pipe_file) > + > + autogen_rt, errorcode = self.StartAutoGen(mqueue, > + Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR) > + > + # Skip cache hit modules > + if GlobalData.gBinCacheSource: > + for Ma in TotalModules: > + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ > + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit: > + self.HashSkipModules.append(Ma) > + continue > + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ > + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit: > + self.HashSkipModules.append(Ma) > continue > - if Ma.PcdIsDriver: > - Ma.PlatformInfo = Pa > - Ma.Workspace = Wa > - PcdMaList.append(Ma) > - TotalModules.append(Ma) > - # Initialize all modules in tracking to 'FAIL' > - GlobalData.gModuleBuildTracking[Ma] = 'FAIL' > + BuildModules.append(Ma) > + else: > + BuildModules.extend(TotalModules) > > - mqueue = mp.Queue() > - for m in Pa.GetAllModuleInfo: > - mqueue.put(m) > - data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) > - Pa.DataPipe.dump(data_pipe_file) > - autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, > GlobalData.gCacheIR) > + if not autogen_rt: > + self.AutoGenMgr.TerminateWorkers() > + self.AutoGenMgr.join(1) > + raise FatalError(errorcode) > + self.AutoGenTime += int(round((time.time() - AutoGenStart))) > + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") > + with open(AutoGenIdFile,"w") as fw: > + fw.write("Arch=%s\n" % "|".join((Wa.ArchList))) > + fw.write("BuildDir=%s\n" % Wa.BuildDir) > + fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid)) > + self.Progress.Stop("done!") > + return Wa, BuildModules > > - # Skip cache hit modules > - if GlobalData.gBinCacheSource: > - for Ma in TotalModules: > - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ > - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit: > - self.HashSkipModules.append(Ma) > - continue > - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ > - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit: > - self.HashSkipModules.append(Ma) > - continue > - self.BuildModules.append(Ma) > + def _MultiThreadBuildPlatform(self): > + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False) > + for BuildTarget in self.BuildTargetList: > + GlobalData.gGlobalDefines['TARGET'] = BuildTarget > + index = 0 > + for ToolChain in self.ToolChainList: > + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain > + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain > + GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index] > + index += 1 > + ExitFlag = threading.Event() > + ExitFlag.clear() > + if self.SkipAutoGen: > + Wa = self.VerifyAutoGenFiles() > + if Wa is None: > + Wa, self.BuildModules = > + self.PerformAutoGen(BuildTarget,ToolChain) > else: > - self.BuildModules.extend(TotalModules) > - > - if not autogen_rt: > - self.AutoGenMgr.TerminateWorkers() > - self.AutoGenMgr.join(0.1) > - raise FatalError(errorcode) > - self.AutoGenTime += int(round((time.time() - AutoGenStart))) > - self.Progress.Stop("done!") > + GlobalData.gAutoGenPhase = True > + self.BuildModules = self.SetupMakeSetting(Wa) > + else: > + Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain) > + Pa = Wa.AutoGenObjectList[0] > + GlobalData.gAutoGenPhase = False > > if GlobalData.gBinCacheSource: > EdkLogger.quiet("Total cache hit driver num: %s, > cache miss driver num: %s" % (len(set(self.HashSkipModules)), > len(set(self.BuildModules)))) > CacheHitMa = set() > CacheNotHitMa = set() @@ -2136,18 +2246,17 @@ > class Build(): > # Start task scheduler > if not BuildTask.IsOnGoing(): > > BuildTask.StartScheduler(self.ThreadNumber, ExitFlag) > > # in case there's an interruption. we need a full version of makefile for platform > - Pa.CreateMakeFile(False) > + > if BuildTask.HasError(): > self.invalidateHash() > EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) > self.MakeTime += int(round((time.time() - > MakeStart))) > > MakeContiue = time.time() > - > # > # > # All modules have been put in build tasks queue. Tell task scheduler > # to exit if all tasks are completed > # > @@ -2175,17 +2284,12 @@ class Build(): > if (Arch == 'IA32' or Arch == 'ARM') and > self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000: > EdkLogger.error("build", > PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules") > # > # Get Module List > # > - ModuleList = {} > - for Pa in Wa.AutoGenObjectList: > - for Ma in Pa.ModuleAutoGenList: > - if Ma is None: > - continue > - if not Ma.IsLibrary: > - ModuleList[Ma.Guid.upper()] = Ma > + ModuleList = {ma.Guid.upper():ma for ma in > + self.BuildModules} > + > # > # Rebase module to the preferred memory address before GenFds > # > MapBuffer = [] > if self.LoadFixAddress != 0: > @@ -2206,33 +2310,17 @@ class Build(): > self.GenFdsTime += int(round((time.time() - GenFdsStart))) > # > # Save MAP buffer into MAP file. > # > self._SaveMapFile(MapBuffer, Wa) > + self.CreateGuidedSectionToolsFile(Wa) > self.invalidateHash() > - > ## Generate GuidedSectionTools.txt in the FV directories. > # > - def CreateGuidedSectionToolsFile(self): > + def CreateGuidedSectionToolsFile(self,Wa): > for BuildTarget in self.BuildTargetList: > for ToolChain in self.ToolChainList: > - Wa = WorkspaceAutoGen( > - self.WorkspaceDir, > - self.PlatformFile, > - BuildTarget, > - ToolChain, > - self.ArchList, > - self.BuildDatabase, > - self.TargetTxt, > - self.ToolDef, > - self.Fdf, > - self.FdList, > - self.FvList, > - self.CapList, > - self.SkuId, > - self.UniFlag > - ) > FvDir = Wa.FvDir > if not os.path.exists(FvDir): > continue > > for Arch in self.ArchList: > @@ -2285,11 +2373,10 @@ class Build(): > if not self.SpawnMode or self.Target not in ["", "all"]: > self.SpawnMode = False > self._BuildPlatform() > else: > self._MultiThreadBuildPlatform() > - self.CreateGuidedSectionToolsFile() > else: > self.SpawnMode = False > self._BuildModule() > > if self.Target == 'cleanall': > -- > 2.20.1.windows.1 > > > ^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen 2019-09-05 14:05 ` Bob Feng @ 2019-09-09 6:58 ` Liming Gao [not found] ` <15C2B2E9C55F0B32.16248@groups.io> 1 sibling, 0 replies; 6+ messages in thread From: Liming Gao @ 2019-09-09 6:58 UTC (permalink / raw) To: Feng, Bob C, devel@edk2.groups.io; +Cc: Shi, Steven Liming Gao <liming.gao@intel.com> >-----Original Message----- >From: Feng, Bob C >Sent: Thursday, September 05, 2019 10:06 PM >To: devel@edk2.groups.io; Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming ><liming.gao@intel.com> >Cc: Shi, Steven <steven.shi@intel.com> >Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for >IgnoreAutoGen > >Sorry for the incomplete mail. > >I verified this patch on Ovmf, Emulator, MinPlatforms and Intel Server >Platform. > >Thanks, >Bob > >-----Original Message----- >From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Bob Feng >Sent: Thursday, September 5, 2019 10:04 PM >To: Gao, Liming <liming.gao@intel.com>; devel@edk2.groups.io >Cc: Shi, Steven <steven.shi@intel.com> >Subject: Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for >IgnoreAutoGen > >Liming, > >Yes, this change only impact -u option. > >I separate _MultiThreadBuildPlatform function into two functions and create 2 >new functions specially for IgnoreAutoGen, so the there looks big change for >build.py in this patch. I verified this patch on Ovmf, E > >-Bob > >-----Original Message----- >From: Gao, Liming >Sent: Thursday, September 5, 2019 9:12 PM >To: devel@edk2.groups.io; Feng, Bob C <bob.c.feng@intel.com> >Cc: Shi, Steven <steven.shi@intel.com> >Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for >IgnoreAutoGen > >Bob: > Does this change only impact -u option behavior? > >Thanks >Liming >> -----Original Message----- >> From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of >> Bob Feng >> Sent: Thursday, September 5, 2019 5:05 PM >> To: devel@edk2.groups.io >> Cc: Gao, Liming <liming.gao@intel.com>; Shi, Steven >> <steven.shi@intel.com>; Feng, Bob C <bob.c.feng@intel.com> >> Subject: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for >> IgnoreAutoGen >> >> https://bugzilla.tianocore.org/show_bug.cgi?id=2080 >> >> This patch is to improve build -u option to re-use >> GlobalVar_<platformguid>_<arch>.bin file which is introduced by >> multiple-process-autogen feature. >> >> Cc: Liming Gao <liming.gao@intel.com> >> Cc: Steven Shi <steven.shi@intel.com> >> Signed-off-by: Bob Feng <bob.c.feng@intel.com> >> --- >> .../Source/Python/AutoGen/AutoGenWorker.py | 2 +- >> .../Source/Python/AutoGen/ModuleAutoGen.py | 4 +- >> .../Python/AutoGen/ModuleAutoGenHelper.py | 29 ++ >> .../Source/Python/AutoGen/PlatformAutoGen.py | 2 + >> .../Source/Python/AutoGen/WorkspaceAutoGen.py | 1 - >> BaseTools/Source/Python/build/build.py | 355 +++++++++++------- >> 6 files changed, 255 insertions(+), 138 deletions(-) >> >> diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py >> b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py >> index 2e68538b1cb4..f488ae9d5f80 100755 >> --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py >> +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py >> @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process): >> Ma.ReferenceModules = >Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] >> if GlobalData.gBinCacheSource and CommandTarget in [None, "", >"all"]: >> Ma.GenModuleFilesHash(GlobalData.gCacheIR) >> Ma.GenPreMakefileHash(GlobalData.gCacheIR) >> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): >> - continue >> + continue >> >> Ma.CreateCodeFile(False) >> >> Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, >> Ma.Arch),[])) >> >> if GlobalData.gBinCacheSource and CommandTarget in [None, "", >"all"]: >> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py >> b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py >> index 3bb7e91154ac..5f28681e3146 100755 >> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py >> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py >> @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen): >> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: >> self.GenModuleFilesHash(gDict) >> >> if not (self.MetaFile.Path, self.Arch) in gDict or \ >> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: >> - EdkLogger.quiet("[cache warning]: Cannot generate >ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, >> self.Arch)) >> - return >> + EdkLogger.quiet("[cache warning]: Cannot generate >> + ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, >> self.Arch)) >> + return >> >> # Initialze hash object >> m = hashlib.md5() >> >> # Add Platform level hash >> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py >> b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py >> index c7591253debd..9dd93b9beb12 100644 >> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py >> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py >> @@ -234,22 +234,51 @@ class AutoGenInfo(object): >> # >> #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in >> multiple thread # class WorkSpaceInfo(AutoGenInfo): >> def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch): >> + if not hasattr(self, "_Init"): >> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch) >> + self._Init = True >> + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch): >> self._SrcTimeStamp = 0 >> self.Db = BuildDB >> self.BuildDatabase = self.Db.BuildObject >> self.Target = Target >> self.ToolChain = ToolChain >> self.WorkspaceDir = Workspace >> self.ActivePlatform = MetaFile >> self.ArchList = Arch >> + self.AutoGenObjectList = [] >> + @property >> + def BuildDir(self): >> + return self.AutoGenObjectList[0].BuildDir >> >> + @property >> + def Name(self): >> + return self.AutoGenObjectList[0].Platform.PlatformName >> + >> + @property >> + def FlashDefinition(self): >> + return self.AutoGenObjectList[0].Platform.FlashDefinition >> + @property >> + def GenFdsCommandDict(self): >> + FdsCommandDict = >self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict") >> + if FdsCommandDict: >> + return FdsCommandDict >> + return {} >> + >> + @cached_property >> + def FvDir(self): >> + return os.path.join(self.BuildDir, TAB_FV_DIRECTORY) >> >> class PlatformInfo(AutoGenInfo): >> def __init__(self, Workspace, MetaFile, Target, ToolChain, Arch,DataPipe): >> + if not hasattr(self, "_Init"): >> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe) >> + self._Init = True >> + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe): >> self.Wa = Workspace >> self.WorkspaceDir = self.Wa.WorkspaceDir >> self.MetaFile = MetaFile >> self.Arch = Arch >> self.Target = Target >> diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py >> b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py >> index 565424a95ead..1e17b6687129 100644 >> --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py >> +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py >> @@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen): >> key = (Ma.MetaFile.File, self.Arch) >> if key in FfsCommand: >> Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key]) >> else: >> Ma.CreateMakeFile(CreateModuleMakeFile) >> + self.CreateLibModuelDirs() >> >> + def CreateLibModuelDirs(self): >> # no need to create makefile for the platform more than once >> if self.IsMakeFileCreated: >> return >> >> # create library/module build dirs for platform diff --git >> a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py >> b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py >> index 24942674721f..f9d2c216169c 100644 >> --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py >> +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py >> @@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen): >> self._CheckPcdDefineAndType() >> >> self.CreateBuildOptionsFile() >> self.CreatePcdTokenNumberFile() >> self.CreateModuleHashInfo() >> - GlobalData.gAutoGenPhase = False >> >> # >> # Merge Arch >> # >> def MergeArch(self): >> diff --git a/BaseTools/Source/Python/build/build.py >> b/BaseTools/Source/Python/build/build.py >> index 0406ac314b65..db57c3e526da 100755 >> --- a/BaseTools/Source/Python/build/build.py >> +++ b/BaseTools/Source/Python/build/build.py >> @@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import >> PatchBinaryFile >> >> import Common.GlobalData as GlobalData from GenFds.GenFds import >> GenFds, GenFdsApi import multiprocessing as mp from multiprocessing >> import Manager >> +from AutoGen.DataPipe import MemoryDataPipe from >> +AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo >from >> +GenFds.FdfParser import FdfParser >> >> >> ## standard targets of build command >> gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', >> 'fds', 'clean', 'cleanall', 'cleanlib', 'run'] >> >> @@ -837,11 +840,11 @@ class Build(): >> for PcdMa in PcdMaList: >> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: >> PcdMa.GenModuleFilesHash(share_data) >> PcdMa.GenPreMakefileHash(share_data) >> if PcdMa.CanSkipbyPreMakefileCache(share_data): >> - continue >> + continue >> >> PcdMa.CreateCodeFile(False) >> PcdMa.CreateMakeFile(False,GenFfsList = >> DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, >> PcdMa.Arch),[])) >> >> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: >> @@ -1218,18 +1221,28 @@ class Build(): >> mqueue = mp.Queue() >> for m in AutoGenObject.GetAllModuleInfo: >> mqueue.put(m) >> >> AutoGenObject.DataPipe.DataContainer = {"CommandTarget": >> self.Target} >> + AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": >AutoGenObject.Workspace._SrcTimeStamp} >> + AutoGenObject.CreateLibModuelDirs() >> + AutoGenObject.DataPipe.DataContainer = >{"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList} >> + AutoGenObject.DataPipe.DataContainer = >{"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList} >> + AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": >> + AutoGenObject.Workspace.GenFdsCommandDict} >> self.Progress.Start("Generating makefile and code") >> data_pipe_file = os.path.join(AutoGenObject.BuildDir, >> "GlobalVar_%s_%s.bin" % >> (str(AutoGenObject.Guid),AutoGenObject.Arch)) >> AutoGenObject.DataPipe.dump(data_pipe_file) >> autogen_rt,errorcode = self.StartAutoGen(mqueue, >> AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, >> GlobalData.gCacheIR) >> + AutoGenIdFile = >os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") >> + with open(AutoGenIdFile,"w") as fw: >> + fw.write("Arch=%s\n" % >"|".join((AutoGenObject.Workspace.ArchList))) >> + fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir) >> + fw.write("PlatformGuid=%s\n" % >> + str(AutoGenObject.Guid)) >> self.Progress.Stop("done!") >> if not autogen_rt: >> self.AutoGenMgr.TerminateWorkers() >> - self.AutoGenMgr.join(0.1) >> + self.AutoGenMgr.join(1) >> raise FatalError(errorcode) >> AutoGenObject.CreateCodeFile(False) >> AutoGenObject.CreateMakeFile(False) >> else: >> # always recreate top/platform makefile when clean, just >> in case of inconsistency @@ -1778,10 +1791,11 @@ class Build(): >> self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList) >> # >> # Save MAP buffer into MAP file. >> # >> self._SaveMapFile (MapBuffer, Wa) >> + self.CreateGuidedSectionToolsFile(Wa) >> >> ## Build active module for different build targets, different tool chains >and different archs >> # >> def _BuildModule(self): >> for BuildTarget in self.BuildTargetList: >> @@ -1852,13 +1866,13 @@ class Build(): >> >> if GlobalData.gBinCacheSource and self.Target in [None, "", >"all"]: >> Ma.GenModuleFilesHash(GlobalData.gCacheIR) >> Ma.GenPreMakefileHash(GlobalData.gCacheIR) >> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): >> - self.HashSkipModules.append(Ma) >> - EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, >Ma.Arch)) >> - continue >> + self.HashSkipModules.append(Ma) >> + EdkLogger.quiet("cache hit: %s[%s]" % >(Ma.MetaFile.Path, Ma.Arch)) >> + continue >> >> # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', >'fds' >> if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: >> # for target which must generate AutoGen code and >makefile >> if not self.SkipAutoGen or self.Target == 'genc': >> @@ -1987,125 +2001,221 @@ class Build(): >> GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, >ArchList, GlobalData) >> for Cmd in GenFfsDict: >> tmpInf, tmpArch = GenFfsDict[Cmd] >> CmdSetDict[tmpInf, tmpArch].add(Cmd) >> return CmdSetDict >> + def VerifyAutoGenFiles(self): >> + AutoGenIdFile = >os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") >> + try: >> + with open(AutoGenIdFile) as fd: >> + lines = fd.readlines() >> + except: >> + return None >> + for line in lines: >> + if "Arch" in line: >> + ArchList = line.strip().split("=")[1].split("|") >> + if "BuildDir" in line: >> + BuildDir = line.split("=")[1].strip() >> + if "PlatformGuid" in line: >> + PlatformGuid = line.split("=")[1].strip() >> + GlobalVarList = [] >> + for arch in ArchList: >> + global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % >(str(PlatformGuid),arch)) >> + if not os.path.exists(global_var): >> + return None >> + GlobalVarList.append(global_var) >> + for global_var in GlobalVarList: >> + data_pipe = MemoryDataPipe() >> + data_pipe.load(global_var) >> + target = data_pipe.Get("P_Info").get("Target") >> + toolchain = data_pipe.Get("P_Info").get("ToolChain") >> + archlist = data_pipe.Get("P_Info").get("ArchList") >> + Arch = data_pipe.Get("P_Info").get("Arch") >> + active_p = data_pipe.Get("P_Info").get("ActivePlatform") >> + workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir") >> + PackagesPath = os.getenv("PACKAGES_PATH") >> + mws.setWs(workspacedir, PackagesPath) >> + LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList") >> + ModuleBuildDirectoryList = >> + data_pipe.Get("ModuleBuildDirectoryList") >> + >> + for m_build_dir in LibraryBuildDirectoryList: >> + if not >os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[Ge >nMake.gMakeType])): >> + return None >> + for m_build_dir in ModuleBuildDirectoryList: >> + if not >os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[Ge >nMake.gMakeType])): >> + return None >> + Wa = WorkSpaceInfo( >> + workspacedir,active_p,target,toolchain,archlist >> + ) >> + Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe) >> + Wa.AutoGenObjectList.append(Pa) >> + return Wa >> + def SetupMakeSetting(self,Wa): >> + BuildModules = [] >> + for Pa in Wa.AutoGenObjectList: >> + for m in Pa._MbList: >> + ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, >Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe) >> + BuildModules.append(ma) >> + fdf_file = Wa.FlashDefinition >> + if fdf_file: >> + Fdf = FdfParser(fdf_file.Path) >> + Fdf.ParseFile() >> + GlobalData.gFdfParser = Fdf >> + if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict: >> + FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName] >> + for FdRegion in FdDict.RegionList: >> + if str(FdRegion.RegionType) is 'FILE' and >self.Platform.VpdToolGuid in str(FdRegion.RegionDataList): >> + if int(FdRegion.Offset) % 8 != 0: >> + EdkLogger.error("build", FORMAT_INVALID, >> + 'The VPD Base Address %s must be 8-byte aligned.' % >> (FdRegion.Offset)) >> + Wa.FdfProfile = Fdf.Profile >> + self.Fdf = Fdf >> + else: >> + self.Fdf = None >> + return BuildModules >> >> ## Build a platform in multi-thread mode >> # >> - def _MultiThreadBuildPlatform(self): >> - SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE >auto-generated\n', False) >> - for BuildTarget in self.BuildTargetList: >> - GlobalData.gGlobalDefines['TARGET'] = BuildTarget >> - index = 0 >> - for ToolChain in self.ToolChainList: >> - WorkspaceAutoGenTime = time.time() >> - GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain >> - GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain >> - GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index] >> - index += 1 >> - Wa = WorkspaceAutoGen( >> - self.WorkspaceDir, >> - self.PlatformFile, >> - BuildTarget, >> - ToolChain, >> - self.ArchList, >> - self.BuildDatabase, >> - self.TargetTxt, >> - self.ToolDef, >> - self.Fdf, >> - self.FdList, >> - self.FvList, >> - self.CapList, >> - self.SkuId, >> - self.UniFlag, >> - self.Progress >> - ) >> - self.Fdf = Wa.FdfFile >> - self.LoadFixAddress = Wa.Platform.LoadFixAddress >> - self.BuildReport.AddPlatformReport(Wa) >> - Wa.CreateMakeFile(False) >> + def PerformAutoGen(self,BuildTarget,ToolChain): >> + WorkspaceAutoGenTime = time.time() >> + Wa = WorkspaceAutoGen( >> + self.WorkspaceDir, >> + self.PlatformFile, >> + BuildTarget, >> + ToolChain, >> + self.ArchList, >> + self.BuildDatabase, >> + self.TargetTxt, >> + self.ToolDef, >> + self.Fdf, >> + self.FdList, >> + self.FvList, >> + self.CapList, >> + self.SkuId, >> + self.UniFlag, >> + self.Progress >> + ) >> + self.Fdf = Wa.FdfFile >> + self.LoadFixAddress = Wa.Platform.LoadFixAddress >> + self.BuildReport.AddPlatformReport(Wa) >> + Wa.CreateMakeFile(False) >> >> # Add ffs build to makefile >> - CmdListDict = {} >> - if GlobalData.gEnableGenfdsMultiThread and self.Fdf: >> - CmdListDict = self._GenFfsCmd(Wa.ArchList) >> + CmdListDict = {} >> + if GlobalData.gEnableGenfdsMultiThread and self.Fdf: >> + CmdListDict = self._GenFfsCmd(Wa.ArchList) >> >> - # Add Platform and Package level hash in share_data for module >hash calculation later >> - if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: >> - GlobalData.gCacheIR[('PlatformHash')] = >GlobalData.gPlatformHash >> - for PkgName in GlobalData.gPackageHash.keys(): >> - GlobalData.gCacheIR[(PkgName, 'PackageHash')] = >GlobalData.gPackageHash[PkgName] >> + # Add Platform and Package level hash in share_data for module hash >calculation later >> + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: >> + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash >> + for PkgName in GlobalData.gPackageHash.keys(): >> + GlobalData.gCacheIR[(PkgName, 'PackageHash')] = >> + GlobalData.gPackageHash[PkgName] >> >> - # multi-thread exit flag >> - ExitFlag = threading.Event() >> - ExitFlag.clear() >> - self.AutoGenTime += int(round((time.time() - >WorkspaceAutoGenTime))) >> - self.BuildModules = [] >> - TotalModules = [] >> - for Arch in Wa.ArchList: >> - PcdMaList = [] >> - AutoGenStart = time.time() >> - GlobalData.gGlobalDefines['ARCH'] = Arch >> - Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, >ToolChain, Arch) >> - if Pa is None: >> - continue >> - ModuleList = [] >> - for Inf in Pa.Platform.Modules: >> - ModuleList.append(Inf) >> + self.AutoGenTime += int(round((time.time() - >WorkspaceAutoGenTime))) >> + BuildModules = [] >> + TotalModules = [] >> + for Arch in Wa.ArchList: >> + PcdMaList = [] >> + AutoGenStart = time.time() >> + GlobalData.gGlobalDefines['ARCH'] = Arch >> + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, >Arch) >> + if Pa is None: >> + continue >> + ModuleList = [] >> + for Inf in Pa.Platform.Modules: >> + ModuleList.append(Inf) >> # Add the INF only list in FDF >> - if GlobalData.gFdfParser is not None: >> - for InfName in GlobalData.gFdfParser.Profile.InfList: >> - Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch) >> - if Inf in Pa.Platform.Modules: >> - continue >> - ModuleList.append(Inf) >> - Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} >> - Pa.DataPipe.DataContainer = {"Workspace_timestamp": >Wa._SrcTimeStamp} >> - Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} >> - for Module in ModuleList: >> + if GlobalData.gFdfParser is not None: >> + for InfName in GlobalData.gFdfParser.Profile.InfList: >> + Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch) >> + if Inf in Pa.Platform.Modules: >> + continue >> + ModuleList.append(Inf) >> + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} >> + Pa.DataPipe.DataContainer = {"Workspace_timestamp": >Wa._SrcTimeStamp} >> + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} >> + Pa.CreateLibModuelDirs() >> + Pa.DataPipe.DataContainer = >{"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList} >> + Pa.DataPipe.DataContainer = >{"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList} >> + Pa.DataPipe.DataContainer = {"FdsCommandDict": >Wa.GenFdsCommandDict} >> + ModuleCodaFile = {} >> + for ma in Pa.ModuleAutoGenList: >> + >> + ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFil >> + e.Path)] = [item.Target for item in >> ma.CodaTargetList] >> + Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile} >> + for Module in ModuleList: >> # Get ModuleAutoGen object to generate C code file and >makefile >> - Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, >Arch, self.PlatformFile,Pa.DataPipe) >> + Ma = ModuleAutoGen(Wa, Module, BuildTarget, >> + ToolChain, Arch, self.PlatformFile,Pa.DataPipe) >> >> - if Ma is None: >> + if Ma is None: >> + continue >> + if Ma.PcdIsDriver: >> + Ma.PlatformInfo = Pa >> + Ma.Workspace = Wa >> + PcdMaList.append(Ma) >> + TotalModules.append(Ma) >> + # Initialize all modules in tracking to 'FAIL' >> + GlobalData.gModuleBuildTracking[Ma] = 'FAIL' >> + >> + >> + mqueue = mp.Queue() >> + for m in Pa.GetAllModuleInfo: >> + mqueue.put(m) >> + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % >(str(Pa.Guid),Pa.Arch)) >> + Pa.DataPipe.dump(data_pipe_file) >> + >> + autogen_rt, errorcode = self.StartAutoGen(mqueue, >> + Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR) >> + >> + # Skip cache hit modules >> + if GlobalData.gBinCacheSource: >> + for Ma in TotalModules: >> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ >> + GlobalData.gCacheIR[(Ma.MetaFile.Path, >Ma.Arch)].PreMakeCacheHit: >> + self.HashSkipModules.append(Ma) >> + continue >> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ >> + GlobalData.gCacheIR[(Ma.MetaFile.Path, >Ma.Arch)].MakeCacheHit: >> + self.HashSkipModules.append(Ma) >> continue >> - if Ma.PcdIsDriver: >> - Ma.PlatformInfo = Pa >> - Ma.Workspace = Wa >> - PcdMaList.append(Ma) >> - TotalModules.append(Ma) >> - # Initialize all modules in tracking to 'FAIL' >> - GlobalData.gModuleBuildTracking[Ma] = 'FAIL' >> + BuildModules.append(Ma) >> + else: >> + BuildModules.extend(TotalModules) >> >> - mqueue = mp.Queue() >> - for m in Pa.GetAllModuleInfo: >> - mqueue.put(m) >> - data_pipe_file = os.path.join(Pa.BuildDir, >"GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) >> - Pa.DataPipe.dump(data_pipe_file) >> - autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, >self.SkipAutoGen, PcdMaList, >> GlobalData.gCacheIR) >> + if not autogen_rt: >> + self.AutoGenMgr.TerminateWorkers() >> + self.AutoGenMgr.join(1) >> + raise FatalError(errorcode) >> + self.AutoGenTime += int(round((time.time() - AutoGenStart))) >> + AutoGenIdFile = >os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") >> + with open(AutoGenIdFile,"w") as fw: >> + fw.write("Arch=%s\n" % "|".join((Wa.ArchList))) >> + fw.write("BuildDir=%s\n" % Wa.BuildDir) >> + fw.write("PlatformGuid=%s\n" % >str(Wa.AutoGenObjectList[0].Guid)) >> + self.Progress.Stop("done!") >> + return Wa, BuildModules >> >> - # Skip cache hit modules >> - if GlobalData.gBinCacheSource: >> - for Ma in TotalModules: >> - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ >> - GlobalData.gCacheIR[(Ma.MetaFile.Path, >Ma.Arch)].PreMakeCacheHit: >> - self.HashSkipModules.append(Ma) >> - continue >> - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ >> - GlobalData.gCacheIR[(Ma.MetaFile.Path, >Ma.Arch)].MakeCacheHit: >> - self.HashSkipModules.append(Ma) >> - continue >> - self.BuildModules.append(Ma) >> + def _MultiThreadBuildPlatform(self): >> + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE >auto-generated\n', False) >> + for BuildTarget in self.BuildTargetList: >> + GlobalData.gGlobalDefines['TARGET'] = BuildTarget >> + index = 0 >> + for ToolChain in self.ToolChainList: >> + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain >> + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain >> + GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index] >> + index += 1 >> + ExitFlag = threading.Event() >> + ExitFlag.clear() >> + if self.SkipAutoGen: >> + Wa = self.VerifyAutoGenFiles() >> + if Wa is None: >> + Wa, self.BuildModules = >> + self.PerformAutoGen(BuildTarget,ToolChain) >> else: >> - self.BuildModules.extend(TotalModules) >> - >> - if not autogen_rt: >> - self.AutoGenMgr.TerminateWorkers() >> - self.AutoGenMgr.join(0.1) >> - raise FatalError(errorcode) >> - self.AutoGenTime += int(round((time.time() - AutoGenStart))) >> - self.Progress.Stop("done!") >> + GlobalData.gAutoGenPhase = True >> + self.BuildModules = self.SetupMakeSetting(Wa) >> + else: >> + Wa, self.BuildModules = >self.PerformAutoGen(BuildTarget,ToolChain) >> + Pa = Wa.AutoGenObjectList[0] >> + GlobalData.gAutoGenPhase = False >> >> if GlobalData.gBinCacheSource: >> EdkLogger.quiet("Total cache hit driver num: %s, >> cache miss driver num: %s" % (len(set(self.HashSkipModules)), >> len(set(self.BuildModules)))) >> CacheHitMa = set() >> CacheNotHitMa = set() @@ -2136,18 +2246,17 @@ >> class Build(): >> # Start task scheduler >> if not BuildTask.IsOnGoing(): >> >> BuildTask.StartScheduler(self.ThreadNumber, ExitFlag) >> >> # in case there's an interruption. we need a full version of >makefile for platform >> - Pa.CreateMakeFile(False) >> + >> if BuildTask.HasError(): >> self.invalidateHash() >> EdkLogger.error("build", BUILD_ERROR, "Failed to build >module", ExtraData=GlobalData.gBuildingModule) >> self.MakeTime += int(round((time.time() - >> MakeStart))) >> >> MakeContiue = time.time() >> - >> # >> # >> # All modules have been put in build tasks queue. Tell task >scheduler >> # to exit if all tasks are completed >> # >> @@ -2175,17 +2284,12 @@ class Build(): >> if (Arch == 'IA32' or Arch == 'ARM') and >> self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= >0x100000000: >> EdkLogger.error("build", >> PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to >larger than or equal to 4G for the platorm with IA32 or ARM arch modules") >> # >> # Get Module List >> # >> - ModuleList = {} >> - for Pa in Wa.AutoGenObjectList: >> - for Ma in Pa.ModuleAutoGenList: >> - if Ma is None: >> - continue >> - if not Ma.IsLibrary: >> - ModuleList[Ma.Guid.upper()] = Ma >> + ModuleList = {ma.Guid.upper():ma for ma in >> + self.BuildModules} >> + >> # >> # Rebase module to the preferred memory address before >GenFds >> # >> MapBuffer = [] >> if self.LoadFixAddress != 0: >> @@ -2206,33 +2310,17 @@ class Build(): >> self.GenFdsTime += int(round((time.time() - GenFdsStart))) >> # >> # Save MAP buffer into MAP file. >> # >> self._SaveMapFile(MapBuffer, Wa) >> + self.CreateGuidedSectionToolsFile(Wa) >> self.invalidateHash() >> - >> ## Generate GuidedSectionTools.txt in the FV directories. >> # >> - def CreateGuidedSectionToolsFile(self): >> + def CreateGuidedSectionToolsFile(self,Wa): >> for BuildTarget in self.BuildTargetList: >> for ToolChain in self.ToolChainList: >> - Wa = WorkspaceAutoGen( >> - self.WorkspaceDir, >> - self.PlatformFile, >> - BuildTarget, >> - ToolChain, >> - self.ArchList, >> - self.BuildDatabase, >> - self.TargetTxt, >> - self.ToolDef, >> - self.Fdf, >> - self.FdList, >> - self.FvList, >> - self.CapList, >> - self.SkuId, >> - self.UniFlag >> - ) >> FvDir = Wa.FvDir >> if not os.path.exists(FvDir): >> continue >> >> for Arch in self.ArchList: >> @@ -2285,11 +2373,10 @@ class Build(): >> if not self.SpawnMode or self.Target not in ["", "all"]: >> self.SpawnMode = False >> self._BuildPlatform() >> else: >> self._MultiThreadBuildPlatform() >> - self.CreateGuidedSectionToolsFile() >> else: >> self.SpawnMode = False >> self._BuildModule() >> >> if self.Target == 'cleanall': >> -- >> 2.20.1.windows.1 >> >> >> > > > ^ permalink raw reply [flat|nested] 6+ messages in thread
[parent not found: <15C2B2E9C55F0B32.16248@groups.io>]
* Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen [not found] ` <15C2B2E9C55F0B32.16248@groups.io> @ 2019-09-09 7:15 ` Liming Gao 0 siblings, 0 replies; 6+ messages in thread From: Liming Gao @ 2019-09-09 7:15 UTC (permalink / raw) To: devel@edk2.groups.io, Gao, Liming, Feng, Bob C; +Cc: Shi, Steven Sorry, should be Acked-by: Liming Gao <liming.gao@intel.com> >-----Original Message----- >From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of >Liming Gao >Sent: Monday, September 09, 2019 2:59 PM >To: Feng, Bob C <bob.c.feng@intel.com>; devel@edk2.groups.io >Cc: Shi, Steven <steven.shi@intel.com> >Subject: Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for >IgnoreAutoGen > >Liming Gao <liming.gao@intel.com> > >>-----Original Message----- >>From: Feng, Bob C >>Sent: Thursday, September 05, 2019 10:06 PM >>To: devel@edk2.groups.io; Feng, Bob C <bob.c.feng@intel.com>; Gao, >Liming >><liming.gao@intel.com> >>Cc: Shi, Steven <steven.shi@intel.com> >>Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for >>IgnoreAutoGen >> >>Sorry for the incomplete mail. >> >>I verified this patch on Ovmf, Emulator, MinPlatforms and Intel Server >>Platform. >> >>Thanks, >>Bob >> >>-----Original Message----- >>From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Bob >Feng >>Sent: Thursday, September 5, 2019 10:04 PM >>To: Gao, Liming <liming.gao@intel.com>; devel@edk2.groups.io >>Cc: Shi, Steven <steven.shi@intel.com> >>Subject: Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for >>IgnoreAutoGen >> >>Liming, >> >>Yes, this change only impact -u option. >> >>I separate _MultiThreadBuildPlatform function into two functions and create >2 >>new functions specially for IgnoreAutoGen, so the there looks big change >for >>build.py in this patch. I verified this patch on Ovmf, E >> >>-Bob >> >>-----Original Message----- >>From: Gao, Liming >>Sent: Thursday, September 5, 2019 9:12 PM >>To: devel@edk2.groups.io; Feng, Bob C <bob.c.feng@intel.com> >>Cc: Shi, Steven <steven.shi@intel.com> >>Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for >>IgnoreAutoGen >> >>Bob: >> Does this change only impact -u option behavior? >> >>Thanks >>Liming >>> -----Original Message----- >>> From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of >>> Bob Feng >>> Sent: Thursday, September 5, 2019 5:05 PM >>> To: devel@edk2.groups.io >>> Cc: Gao, Liming <liming.gao@intel.com>; Shi, Steven >>> <steven.shi@intel.com>; Feng, Bob C <bob.c.feng@intel.com> >>> Subject: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for >>> IgnoreAutoGen >>> >>> https://bugzilla.tianocore.org/show_bug.cgi?id=2080 >>> >>> This patch is to improve build -u option to re-use >>> GlobalVar_<platformguid>_<arch>.bin file which is introduced by >>> multiple-process-autogen feature. >>> >>> Cc: Liming Gao <liming.gao@intel.com> >>> Cc: Steven Shi <steven.shi@intel.com> >>> Signed-off-by: Bob Feng <bob.c.feng@intel.com> >>> --- >>> .../Source/Python/AutoGen/AutoGenWorker.py | 2 +- >>> .../Source/Python/AutoGen/ModuleAutoGen.py | 4 +- >>> .../Python/AutoGen/ModuleAutoGenHelper.py | 29 ++ >>> .../Source/Python/AutoGen/PlatformAutoGen.py | 2 + >>> .../Source/Python/AutoGen/WorkspaceAutoGen.py | 1 - >>> BaseTools/Source/Python/build/build.py | 355 +++++++++++------- >>> 6 files changed, 255 insertions(+), 138 deletions(-) >>> >>> diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py >>> b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py >>> index 2e68538b1cb4..f488ae9d5f80 100755 >>> --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py >>> +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py >>> @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process): >>> Ma.ReferenceModules = >>Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] >>> if GlobalData.gBinCacheSource and CommandTarget in [None, "", >>"all"]: >>> Ma.GenModuleFilesHash(GlobalData.gCacheIR) >>> Ma.GenPreMakefileHash(GlobalData.gCacheIR) >>> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): >>> - continue >>> + continue >>> >>> Ma.CreateCodeFile(False) >>> >>> Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, >>> Ma.Arch),[])) >>> >>> if GlobalData.gBinCacheSource and CommandTarget in [None, "", >>"all"]: >>> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py >>> b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py >>> index 3bb7e91154ac..5f28681e3146 100755 >>> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py >>> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py >>> @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen): >>> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: >>> self.GenModuleFilesHash(gDict) >>> >>> if not (self.MetaFile.Path, self.Arch) in gDict or \ >>> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: >>> - EdkLogger.quiet("[cache warning]: Cannot generate >>ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, >>> self.Arch)) >>> - return >>> + EdkLogger.quiet("[cache warning]: Cannot generate >>> + ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, >>> self.Arch)) >>> + return >>> >>> # Initialze hash object >>> m = hashlib.md5() >>> >>> # Add Platform level hash >>> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py >>> b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py >>> index c7591253debd..9dd93b9beb12 100644 >>> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py >>> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py >>> @@ -234,22 +234,51 @@ class AutoGenInfo(object): >>> # >>> #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in >>> multiple thread # class WorkSpaceInfo(AutoGenInfo): >>> def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch): >>> + if not hasattr(self, "_Init"): >>> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch) >>> + self._Init = True >>> + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch): >>> self._SrcTimeStamp = 0 >>> self.Db = BuildDB >>> self.BuildDatabase = self.Db.BuildObject >>> self.Target = Target >>> self.ToolChain = ToolChain >>> self.WorkspaceDir = Workspace >>> self.ActivePlatform = MetaFile >>> self.ArchList = Arch >>> + self.AutoGenObjectList = [] >>> + @property >>> + def BuildDir(self): >>> + return self.AutoGenObjectList[0].BuildDir >>> >>> + @property >>> + def Name(self): >>> + return self.AutoGenObjectList[0].Platform.PlatformName >>> + >>> + @property >>> + def FlashDefinition(self): >>> + return self.AutoGenObjectList[0].Platform.FlashDefinition >>> + @property >>> + def GenFdsCommandDict(self): >>> + FdsCommandDict = >>self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict") >>> + if FdsCommandDict: >>> + return FdsCommandDict >>> + return {} >>> + >>> + @cached_property >>> + def FvDir(self): >>> + return os.path.join(self.BuildDir, TAB_FV_DIRECTORY) >>> >>> class PlatformInfo(AutoGenInfo): >>> def __init__(self, Workspace, MetaFile, Target, ToolChain, >Arch,DataPipe): >>> + if not hasattr(self, "_Init"): >>> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe) >>> + self._Init = True >>> + def do_init(self,Workspace, MetaFile, Target, ToolChain, >Arch,DataPipe): >>> self.Wa = Workspace >>> self.WorkspaceDir = self.Wa.WorkspaceDir >>> self.MetaFile = MetaFile >>> self.Arch = Arch >>> self.Target = Target >>> diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py >>> b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py >>> index 565424a95ead..1e17b6687129 100644 >>> --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py >>> +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py >>> @@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen): >>> key = (Ma.MetaFile.File, self.Arch) >>> if key in FfsCommand: >>> Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key]) >>> else: >>> Ma.CreateMakeFile(CreateModuleMakeFile) >>> + self.CreateLibModuelDirs() >>> >>> + def CreateLibModuelDirs(self): >>> # no need to create makefile for the platform more than once >>> if self.IsMakeFileCreated: >>> return >>> >>> # create library/module build dirs for platform diff --git >>> a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py >>> b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py >>> index 24942674721f..f9d2c216169c 100644 >>> --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py >>> +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py >>> @@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen): >>> self._CheckPcdDefineAndType() >>> >>> self.CreateBuildOptionsFile() >>> self.CreatePcdTokenNumberFile() >>> self.CreateModuleHashInfo() >>> - GlobalData.gAutoGenPhase = False >>> >>> # >>> # Merge Arch >>> # >>> def MergeArch(self): >>> diff --git a/BaseTools/Source/Python/build/build.py >>> b/BaseTools/Source/Python/build/build.py >>> index 0406ac314b65..db57c3e526da 100755 >>> --- a/BaseTools/Source/Python/build/build.py >>> +++ b/BaseTools/Source/Python/build/build.py >>> @@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import >>> PatchBinaryFile >>> >>> import Common.GlobalData as GlobalData from GenFds.GenFds import >>> GenFds, GenFdsApi import multiprocessing as mp from multiprocessing >>> import Manager >>> +from AutoGen.DataPipe import MemoryDataPipe from >>> +AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo >>from >>> +GenFds.FdfParser import FdfParser >>> >>> >>> ## standard targets of build command >>> gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', >>> 'fds', 'clean', 'cleanall', 'cleanlib', 'run'] >>> >>> @@ -837,11 +840,11 @@ class Build(): >>> for PcdMa in PcdMaList: >>> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: >>> PcdMa.GenModuleFilesHash(share_data) >>> PcdMa.GenPreMakefileHash(share_data) >>> if PcdMa.CanSkipbyPreMakefileCache(share_data): >>> - continue >>> + continue >>> >>> PcdMa.CreateCodeFile(False) >>> PcdMa.CreateMakeFile(False,GenFfsList = >>> DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, >>> PcdMa.Arch),[])) >>> >>> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: >>> @@ -1218,18 +1221,28 @@ class Build(): >>> mqueue = mp.Queue() >>> for m in AutoGenObject.GetAllModuleInfo: >>> mqueue.put(m) >>> >>> AutoGenObject.DataPipe.DataContainer = {"CommandTarget": >>> self.Target} >>> + AutoGenObject.DataPipe.DataContainer = >{"Workspace_timestamp": >>AutoGenObject.Workspace._SrcTimeStamp} >>> + AutoGenObject.CreateLibModuelDirs() >>> + AutoGenObject.DataPipe.DataContainer = >>{"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList} >>> + AutoGenObject.DataPipe.DataContainer = >>{"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList} >>> + AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": >>> + AutoGenObject.Workspace.GenFdsCommandDict} >>> self.Progress.Start("Generating makefile and code") >>> data_pipe_file = os.path.join(AutoGenObject.BuildDir, >>> "GlobalVar_%s_%s.bin" % >>> (str(AutoGenObject.Guid),AutoGenObject.Arch)) >>> AutoGenObject.DataPipe.dump(data_pipe_file) >>> autogen_rt,errorcode = self.StartAutoGen(mqueue, >>> AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, >>> GlobalData.gCacheIR) >>> + AutoGenIdFile = >>os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") >>> + with open(AutoGenIdFile,"w") as fw: >>> + fw.write("Arch=%s\n" % >>"|".join((AutoGenObject.Workspace.ArchList))) >>> + fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir) >>> + fw.write("PlatformGuid=%s\n" % >>> + str(AutoGenObject.Guid)) >>> self.Progress.Stop("done!") >>> if not autogen_rt: >>> self.AutoGenMgr.TerminateWorkers() >>> - self.AutoGenMgr.join(0.1) >>> + self.AutoGenMgr.join(1) >>> raise FatalError(errorcode) >>> AutoGenObject.CreateCodeFile(False) >>> AutoGenObject.CreateMakeFile(False) >>> else: >>> # always recreate top/platform makefile when clean, just >>> in case of inconsistency @@ -1778,10 +1791,11 @@ class Build(): >>> self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList) >>> # >>> # Save MAP buffer into MAP file. >>> # >>> self._SaveMapFile (MapBuffer, Wa) >>> + self.CreateGuidedSectionToolsFile(Wa) >>> >>> ## Build active module for different build targets, different tool chains >>and different archs >>> # >>> def _BuildModule(self): >>> for BuildTarget in self.BuildTargetList: >>> @@ -1852,13 +1866,13 @@ class Build(): >>> >>> if GlobalData.gBinCacheSource and self.Target in [None, "", >>"all"]: >>> Ma.GenModuleFilesHash(GlobalData.gCacheIR) >>> Ma.GenPreMakefileHash(GlobalData.gCacheIR) >>> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): >>> - self.HashSkipModules.append(Ma) >>> - EdkLogger.quiet("cache hit: %s[%s]" % >(Ma.MetaFile.Path, >>Ma.Arch)) >>> - continue >>> + self.HashSkipModules.append(Ma) >>> + EdkLogger.quiet("cache hit: %s[%s]" % >>(Ma.MetaFile.Path, Ma.Arch)) >>> + continue >>> >>> # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', >>'fds' >>> if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: >>> # for target which must generate AutoGen code and >>makefile >>> if not self.SkipAutoGen or self.Target == 'genc': >>> @@ -1987,125 +2001,221 @@ class Build(): >>> GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, >>ArchList, GlobalData) >>> for Cmd in GenFfsDict: >>> tmpInf, tmpArch = GenFfsDict[Cmd] >>> CmdSetDict[tmpInf, tmpArch].add(Cmd) >>> return CmdSetDict >>> + def VerifyAutoGenFiles(self): >>> + AutoGenIdFile = >>os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") >>> + try: >>> + with open(AutoGenIdFile) as fd: >>> + lines = fd.readlines() >>> + except: >>> + return None >>> + for line in lines: >>> + if "Arch" in line: >>> + ArchList = line.strip().split("=")[1].split("|") >>> + if "BuildDir" in line: >>> + BuildDir = line.split("=")[1].strip() >>> + if "PlatformGuid" in line: >>> + PlatformGuid = line.split("=")[1].strip() >>> + GlobalVarList = [] >>> + for arch in ArchList: >>> + global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % >>(str(PlatformGuid),arch)) >>> + if not os.path.exists(global_var): >>> + return None >>> + GlobalVarList.append(global_var) >>> + for global_var in GlobalVarList: >>> + data_pipe = MemoryDataPipe() >>> + data_pipe.load(global_var) >>> + target = data_pipe.Get("P_Info").get("Target") >>> + toolchain = data_pipe.Get("P_Info").get("ToolChain") >>> + archlist = data_pipe.Get("P_Info").get("ArchList") >>> + Arch = data_pipe.Get("P_Info").get("Arch") >>> + active_p = data_pipe.Get("P_Info").get("ActivePlatform") >>> + workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir") >>> + PackagesPath = os.getenv("PACKAGES_PATH") >>> + mws.setWs(workspacedir, PackagesPath) >>> + LibraryBuildDirectoryList = >data_pipe.Get("LibraryBuildDirectoryList") >>> + ModuleBuildDirectoryList = >>> + data_pipe.Get("ModuleBuildDirectoryList") >>> + >>> + for m_build_dir in LibraryBuildDirectoryList: >>> + if not >>os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[G >e >>nMake.gMakeType])): >>> + return None >>> + for m_build_dir in ModuleBuildDirectoryList: >>> + if not >>os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[G >e >>nMake.gMakeType])): >>> + return None >>> + Wa = WorkSpaceInfo( >>> + workspacedir,active_p,target,toolchain,archlist >>> + ) >>> + Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe) >>> + Wa.AutoGenObjectList.append(Pa) >>> + return Wa >>> + def SetupMakeSetting(self,Wa): >>> + BuildModules = [] >>> + for Pa in Wa.AutoGenObjectList: >>> + for m in Pa._MbList: >>> + ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, >>Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe) >>> + BuildModules.append(ma) >>> + fdf_file = Wa.FlashDefinition >>> + if fdf_file: >>> + Fdf = FdfParser(fdf_file.Path) >>> + Fdf.ParseFile() >>> + GlobalData.gFdfParser = Fdf >>> + if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict: >>> + FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName] >>> + for FdRegion in FdDict.RegionList: >>> + if str(FdRegion.RegionType) is 'FILE' and >>self.Platform.VpdToolGuid in str(FdRegion.RegionDataList): >>> + if int(FdRegion.Offset) % 8 != 0: >>> + EdkLogger.error("build", FORMAT_INVALID, >>> + 'The VPD Base Address %s must be 8-byte aligned.' % >>> (FdRegion.Offset)) >>> + Wa.FdfProfile = Fdf.Profile >>> + self.Fdf = Fdf >>> + else: >>> + self.Fdf = None >>> + return BuildModules >>> >>> ## Build a platform in multi-thread mode >>> # >>> - def _MultiThreadBuildPlatform(self): >>> - SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE >>auto-generated\n', False) >>> - for BuildTarget in self.BuildTargetList: >>> - GlobalData.gGlobalDefines['TARGET'] = BuildTarget >>> - index = 0 >>> - for ToolChain in self.ToolChainList: >>> - WorkspaceAutoGenTime = time.time() >>> - GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain >>> - GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain >>> - GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index] >>> - index += 1 >>> - Wa = WorkspaceAutoGen( >>> - self.WorkspaceDir, >>> - self.PlatformFile, >>> - BuildTarget, >>> - ToolChain, >>> - self.ArchList, >>> - self.BuildDatabase, >>> - self.TargetTxt, >>> - self.ToolDef, >>> - self.Fdf, >>> - self.FdList, >>> - self.FvList, >>> - self.CapList, >>> - self.SkuId, >>> - self.UniFlag, >>> - self.Progress >>> - ) >>> - self.Fdf = Wa.FdfFile >>> - self.LoadFixAddress = Wa.Platform.LoadFixAddress >>> - self.BuildReport.AddPlatformReport(Wa) >>> - Wa.CreateMakeFile(False) >>> + def PerformAutoGen(self,BuildTarget,ToolChain): >>> + WorkspaceAutoGenTime = time.time() >>> + Wa = WorkspaceAutoGen( >>> + self.WorkspaceDir, >>> + self.PlatformFile, >>> + BuildTarget, >>> + ToolChain, >>> + self.ArchList, >>> + self.BuildDatabase, >>> + self.TargetTxt, >>> + self.ToolDef, >>> + self.Fdf, >>> + self.FdList, >>> + self.FvList, >>> + self.CapList, >>> + self.SkuId, >>> + self.UniFlag, >>> + self.Progress >>> + ) >>> + self.Fdf = Wa.FdfFile >>> + self.LoadFixAddress = Wa.Platform.LoadFixAddress >>> + self.BuildReport.AddPlatformReport(Wa) >>> + Wa.CreateMakeFile(False) >>> >>> # Add ffs build to makefile >>> - CmdListDict = {} >>> - if GlobalData.gEnableGenfdsMultiThread and self.Fdf: >>> - CmdListDict = self._GenFfsCmd(Wa.ArchList) >>> + CmdListDict = {} >>> + if GlobalData.gEnableGenfdsMultiThread and self.Fdf: >>> + CmdListDict = self._GenFfsCmd(Wa.ArchList) >>> >>> - # Add Platform and Package level hash in share_data for module >>hash calculation later >>> - if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: >>> - GlobalData.gCacheIR[('PlatformHash')] = >>GlobalData.gPlatformHash >>> - for PkgName in GlobalData.gPackageHash.keys(): >>> - GlobalData.gCacheIR[(PkgName, 'PackageHash')] = >>GlobalData.gPackageHash[PkgName] >>> + # Add Platform and Package level hash in share_data for module hash >>calculation later >>> + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: >>> + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash >>> + for PkgName in GlobalData.gPackageHash.keys(): >>> + GlobalData.gCacheIR[(PkgName, 'PackageHash')] = >>> + GlobalData.gPackageHash[PkgName] >>> >>> - # multi-thread exit flag >>> - ExitFlag = threading.Event() >>> - ExitFlag.clear() >>> - self.AutoGenTime += int(round((time.time() - >>WorkspaceAutoGenTime))) >>> - self.BuildModules = [] >>> - TotalModules = [] >>> - for Arch in Wa.ArchList: >>> - PcdMaList = [] >>> - AutoGenStart = time.time() >>> - GlobalData.gGlobalDefines['ARCH'] = Arch >>> - Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, >>ToolChain, Arch) >>> - if Pa is None: >>> - continue >>> - ModuleList = [] >>> - for Inf in Pa.Platform.Modules: >>> - ModuleList.append(Inf) >>> + self.AutoGenTime += int(round((time.time() - >>WorkspaceAutoGenTime))) >>> + BuildModules = [] >>> + TotalModules = [] >>> + for Arch in Wa.ArchList: >>> + PcdMaList = [] >>> + AutoGenStart = time.time() >>> + GlobalData.gGlobalDefines['ARCH'] = Arch >>> + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, >>Arch) >>> + if Pa is None: >>> + continue >>> + ModuleList = [] >>> + for Inf in Pa.Platform.Modules: >>> + ModuleList.append(Inf) >>> # Add the INF only list in FDF >>> - if GlobalData.gFdfParser is not None: >>> - for InfName in GlobalData.gFdfParser.Profile.InfList: >>> - Inf = PathClass(NormPath(InfName), self.WorkspaceDir, >Arch) >>> - if Inf in Pa.Platform.Modules: >>> - continue >>> - ModuleList.append(Inf) >>> - Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} >>> - Pa.DataPipe.DataContainer = {"Workspace_timestamp": >>Wa._SrcTimeStamp} >>> - Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} >>> - for Module in ModuleList: >>> + if GlobalData.gFdfParser is not None: >>> + for InfName in GlobalData.gFdfParser.Profile.InfList: >>> + Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch) >>> + if Inf in Pa.Platform.Modules: >>> + continue >>> + ModuleList.append(Inf) >>> + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} >>> + Pa.DataPipe.DataContainer = {"Workspace_timestamp": >>Wa._SrcTimeStamp} >>> + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} >>> + Pa.CreateLibModuelDirs() >>> + Pa.DataPipe.DataContainer = >>{"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList} >>> + Pa.DataPipe.DataContainer = >>{"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList} >>> + Pa.DataPipe.DataContainer = {"FdsCommandDict": >>Wa.GenFdsCommandDict} >>> + ModuleCodaFile = {} >>> + for ma in Pa.ModuleAutoGenList: >>> + >>> + >ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFil >>> + e.Path)] = [item.Target for item in >>> ma.CodaTargetList] >>> + Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile} >>> + for Module in ModuleList: >>> # Get ModuleAutoGen object to generate C code file and >>makefile >>> - Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, >>Arch, self.PlatformFile,Pa.DataPipe) >>> + Ma = ModuleAutoGen(Wa, Module, BuildTarget, >>> + ToolChain, Arch, self.PlatformFile,Pa.DataPipe) >>> >>> - if Ma is None: >>> + if Ma is None: >>> + continue >>> + if Ma.PcdIsDriver: >>> + Ma.PlatformInfo = Pa >>> + Ma.Workspace = Wa >>> + PcdMaList.append(Ma) >>> + TotalModules.append(Ma) >>> + # Initialize all modules in tracking to 'FAIL' >>> + GlobalData.gModuleBuildTracking[Ma] = 'FAIL' >>> + >>> + >>> + mqueue = mp.Queue() >>> + for m in Pa.GetAllModuleInfo: >>> + mqueue.put(m) >>> + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % >>(str(Pa.Guid),Pa.Arch)) >>> + Pa.DataPipe.dump(data_pipe_file) >>> + >>> + autogen_rt, errorcode = self.StartAutoGen(mqueue, >>> + Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR) >>> + >>> + # Skip cache hit modules >>> + if GlobalData.gBinCacheSource: >>> + for Ma in TotalModules: >>> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ >>> + GlobalData.gCacheIR[(Ma.MetaFile.Path, >>Ma.Arch)].PreMakeCacheHit: >>> + self.HashSkipModules.append(Ma) >>> + continue >>> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ >>> + GlobalData.gCacheIR[(Ma.MetaFile.Path, >>Ma.Arch)].MakeCacheHit: >>> + self.HashSkipModules.append(Ma) >>> continue >>> - if Ma.PcdIsDriver: >>> - Ma.PlatformInfo = Pa >>> - Ma.Workspace = Wa >>> - PcdMaList.append(Ma) >>> - TotalModules.append(Ma) >>> - # Initialize all modules in tracking to 'FAIL' >>> - GlobalData.gModuleBuildTracking[Ma] = 'FAIL' >>> + BuildModules.append(Ma) >>> + else: >>> + BuildModules.extend(TotalModules) >>> >>> - mqueue = mp.Queue() >>> - for m in Pa.GetAllModuleInfo: >>> - mqueue.put(m) >>> - data_pipe_file = os.path.join(Pa.BuildDir, >>"GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) >>> - Pa.DataPipe.dump(data_pipe_file) >>> - autogen_rt, errorcode = self.StartAutoGen(mqueue, >Pa.DataPipe, >>self.SkipAutoGen, PcdMaList, >>> GlobalData.gCacheIR) >>> + if not autogen_rt: >>> + self.AutoGenMgr.TerminateWorkers() >>> + self.AutoGenMgr.join(1) >>> + raise FatalError(errorcode) >>> + self.AutoGenTime += int(round((time.time() - AutoGenStart))) >>> + AutoGenIdFile = >>os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") >>> + with open(AutoGenIdFile,"w") as fw: >>> + fw.write("Arch=%s\n" % "|".join((Wa.ArchList))) >>> + fw.write("BuildDir=%s\n" % Wa.BuildDir) >>> + fw.write("PlatformGuid=%s\n" % >>str(Wa.AutoGenObjectList[0].Guid)) >>> + self.Progress.Stop("done!") >>> + return Wa, BuildModules >>> >>> - # Skip cache hit modules >>> - if GlobalData.gBinCacheSource: >>> - for Ma in TotalModules: >>> - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and >\ >>> - GlobalData.gCacheIR[(Ma.MetaFile.Path, >>Ma.Arch)].PreMakeCacheHit: >>> - self.HashSkipModules.append(Ma) >>> - continue >>> - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and >\ >>> - GlobalData.gCacheIR[(Ma.MetaFile.Path, >>Ma.Arch)].MakeCacheHit: >>> - self.HashSkipModules.append(Ma) >>> - continue >>> - self.BuildModules.append(Ma) >>> + def _MultiThreadBuildPlatform(self): >>> + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE >>auto-generated\n', False) >>> + for BuildTarget in self.BuildTargetList: >>> + GlobalData.gGlobalDefines['TARGET'] = BuildTarget >>> + index = 0 >>> + for ToolChain in self.ToolChainList: >>> + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain >>> + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain >>> + GlobalData.gGlobalDefines['FAMILY'] = >self.ToolChainFamily[index] >>> + index += 1 >>> + ExitFlag = threading.Event() >>> + ExitFlag.clear() >>> + if self.SkipAutoGen: >>> + Wa = self.VerifyAutoGenFiles() >>> + if Wa is None: >>> + Wa, self.BuildModules = >>> + self.PerformAutoGen(BuildTarget,ToolChain) >>> else: >>> - self.BuildModules.extend(TotalModules) >>> - >>> - if not autogen_rt: >>> - self.AutoGenMgr.TerminateWorkers() >>> - self.AutoGenMgr.join(0.1) >>> - raise FatalError(errorcode) >>> - self.AutoGenTime += int(round((time.time() - AutoGenStart))) >>> - self.Progress.Stop("done!") >>> + GlobalData.gAutoGenPhase = True >>> + self.BuildModules = self.SetupMakeSetting(Wa) >>> + else: >>> + Wa, self.BuildModules = >>self.PerformAutoGen(BuildTarget,ToolChain) >>> + Pa = Wa.AutoGenObjectList[0] >>> + GlobalData.gAutoGenPhase = False >>> >>> if GlobalData.gBinCacheSource: >>> EdkLogger.quiet("Total cache hit driver num: %s, >>> cache miss driver num: %s" % (len(set(self.HashSkipModules)), >>> len(set(self.BuildModules)))) >>> CacheHitMa = set() >>> CacheNotHitMa = set() @@ -2136,18 +2246,17 @@ >>> class Build(): >>> # Start task scheduler >>> if not BuildTask.IsOnGoing(): >>> >>> BuildTask.StartScheduler(self.ThreadNumber, ExitFlag) >>> >>> # in case there's an interruption. we need a full version of >>makefile for platform >>> - Pa.CreateMakeFile(False) >>> + >>> if BuildTask.HasError(): >>> self.invalidateHash() >>> EdkLogger.error("build", BUILD_ERROR, "Failed to build >>module", ExtraData=GlobalData.gBuildingModule) >>> self.MakeTime += int(round((time.time() - >>> MakeStart))) >>> >>> MakeContiue = time.time() >>> - >>> # >>> # >>> # All modules have been put in build tasks queue. Tell task >>scheduler >>> # to exit if all tasks are completed >>> # >>> @@ -2175,17 +2284,12 @@ class Build(): >>> if (Arch == 'IA32' or Arch == 'ARM') and >>> self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= >>0x100000000: >>> EdkLogger.error("build", >>> PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set >to >>larger than or equal to 4G for the platorm with IA32 or ARM arch modules") >>> # >>> # Get Module List >>> # >>> - ModuleList = {} >>> - for Pa in Wa.AutoGenObjectList: >>> - for Ma in Pa.ModuleAutoGenList: >>> - if Ma is None: >>> - continue >>> - if not Ma.IsLibrary: >>> - ModuleList[Ma.Guid.upper()] = Ma >>> + ModuleList = {ma.Guid.upper():ma for ma in >>> + self.BuildModules} >>> + >>> # >>> # Rebase module to the preferred memory address before >>GenFds >>> # >>> MapBuffer = [] >>> if self.LoadFixAddress != 0: >>> @@ -2206,33 +2310,17 @@ class Build(): >>> self.GenFdsTime += int(round((time.time() - GenFdsStart))) >>> # >>> # Save MAP buffer into MAP file. >>> # >>> self._SaveMapFile(MapBuffer, Wa) >>> + self.CreateGuidedSectionToolsFile(Wa) >>> self.invalidateHash() >>> - >>> ## Generate GuidedSectionTools.txt in the FV directories. >>> # >>> - def CreateGuidedSectionToolsFile(self): >>> + def CreateGuidedSectionToolsFile(self,Wa): >>> for BuildTarget in self.BuildTargetList: >>> for ToolChain in self.ToolChainList: >>> - Wa = WorkspaceAutoGen( >>> - self.WorkspaceDir, >>> - self.PlatformFile, >>> - BuildTarget, >>> - ToolChain, >>> - self.ArchList, >>> - self.BuildDatabase, >>> - self.TargetTxt, >>> - self.ToolDef, >>> - self.Fdf, >>> - self.FdList, >>> - self.FvList, >>> - self.CapList, >>> - self.SkuId, >>> - self.UniFlag >>> - ) >>> FvDir = Wa.FvDir >>> if not os.path.exists(FvDir): >>> continue >>> >>> for Arch in self.ArchList: >>> @@ -2285,11 +2373,10 @@ class Build(): >>> if not self.SpawnMode or self.Target not in ["", "all"]: >>> self.SpawnMode = False >>> self._BuildPlatform() >>> else: >>> self._MultiThreadBuildPlatform() >>> - self.CreateGuidedSectionToolsFile() >>> else: >>> self.SpawnMode = False >>> self._BuildModule() >>> >>> if self.Target == 'cleanall': >>> -- >>> 2.20.1.windows.1 >>> >>> >>> >> >> >> > > > ^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2019-09-09 7:15 UTC | newest] Thread overview: 6+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2019-09-05 9:04 [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen Bob Feng 2019-09-05 13:11 ` [edk2-devel] " Liming Gao 2019-09-05 14:04 ` Bob Feng [not found] ` <15C18FCC38905317.10639@groups.io> 2019-09-05 14:05 ` Bob Feng 2019-09-09 6:58 ` Liming Gao [not found] ` <15C2B2E9C55F0B32.16248@groups.io> 2019-09-09 7:15 ` Liming Gao
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox