From mboxrd@z Thu Jan 1 00:00:00 1970 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 192.55.52.136, mailfrom: bob.c.feng@intel.com) Received: from mga12.intel.com (mga12.intel.com [192.55.52.136]) by groups.io with SMTP; Thu, 05 Sep 2019 02:05:04 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga006.jf.intel.com ([10.7.209.51]) by fmsmga106.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 05 Sep 2019 02:05:03 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.64,470,1559545200"; d="scan'208";a="187916052" Received: from shwdepsi1121.ccr.corp.intel.com ([10.239.158.47]) by orsmga006.jf.intel.com with ESMTP; 05 Sep 2019 02:05:01 -0700 From: "Bob Feng" To: devel@edk2.groups.io Cc: Liming Gao , Steven Shi , Bob Feng Subject: [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen Date: Thu, 5 Sep 2019 17:04:58 +0800 Message-Id: <20190905090458.22976-1-bob.c.feng@intel.com> X-Mailer: git-send-email 2.20.1.windows.1 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit https://bugzilla.tianocore.org/show_bug.cgi?id=2080 This patch is to improve build -u option to re-use GlobalVar__.bin file which is introduced by multiple-process-autogen feature. Cc: Liming Gao Cc: Steven Shi Signed-off-by: Bob Feng --- .../Source/Python/AutoGen/AutoGenWorker.py | 2 +- .../Source/Python/AutoGen/ModuleAutoGen.py | 4 +- .../Python/AutoGen/ModuleAutoGenHelper.py | 29 ++ .../Source/Python/AutoGen/PlatformAutoGen.py | 2 + .../Source/Python/AutoGen/WorkspaceAutoGen.py | 1 - BaseTools/Source/Python/build/build.py | 355 +++++++++++------- 6 files changed, 255 insertions(+), 138 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py index 2e68538b1cb4..f488ae9d5f80 100755 --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process): Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: Ma.GenModuleFilesHash(GlobalData.gCacheIR) Ma.GenPreMakefileHash(GlobalData.gCacheIR) if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): - continue + continue Ma.CreateCodeFile(False) Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[])) if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py index 3bb7e91154ac..5f28681e3146 100755 --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen): not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: self.GenModuleFilesHash(gDict) if not (self.MetaFile.Path, self.Arch) in gDict or \ not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: - EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, self.Arch)) - return + EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, self.Arch)) + return # Initialze hash object m = hashlib.md5() # Add Platform level hash diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py index c7591253debd..9dd93b9beb12 100644 --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py @@ -234,22 +234,51 @@ class AutoGenInfo(object): # #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in multiple thread # class WorkSpaceInfo(AutoGenInfo): def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch): + if not hasattr(self, "_Init"): + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch) + self._Init = True + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch): self._SrcTimeStamp = 0 self.Db = BuildDB self.BuildDatabase = self.Db.BuildObject self.Target = Target self.ToolChain = ToolChain self.WorkspaceDir = Workspace self.ActivePlatform = MetaFile self.ArchList = Arch + self.AutoGenObjectList = [] + @property + def BuildDir(self): + return self.AutoGenObjectList[0].BuildDir + @property + def Name(self): + return self.AutoGenObjectList[0].Platform.PlatformName + + @property + def FlashDefinition(self): + return self.AutoGenObjectList[0].Platform.FlashDefinition + @property + def GenFdsCommandDict(self): + FdsCommandDict = self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict") + if FdsCommandDict: + return FdsCommandDict + return {} + + @cached_property + def FvDir(self): + return os.path.join(self.BuildDir, TAB_FV_DIRECTORY) class PlatformInfo(AutoGenInfo): def __init__(self, Workspace, MetaFile, Target, ToolChain, Arch,DataPipe): + if not hasattr(self, "_Init"): + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe) + self._Init = True + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe): self.Wa = Workspace self.WorkspaceDir = self.Wa.WorkspaceDir self.MetaFile = MetaFile self.Arch = Arch self.Target = Target diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py index 565424a95ead..1e17b6687129 100644 --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py @@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen): key = (Ma.MetaFile.File, self.Arch) if key in FfsCommand: Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key]) else: Ma.CreateMakeFile(CreateModuleMakeFile) + self.CreateLibModuelDirs() + def CreateLibModuelDirs(self): # no need to create makefile for the platform more than once if self.IsMakeFileCreated: return # create library/module build dirs for platform diff --git a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py index 24942674721f..f9d2c216169c 100644 --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py @@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen): self._CheckPcdDefineAndType() self.CreateBuildOptionsFile() self.CreatePcdTokenNumberFile() self.CreateModuleHashInfo() - GlobalData.gAutoGenPhase = False # # Merge Arch # def MergeArch(self): diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index 0406ac314b65..db57c3e526da 100755 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import PatchBinaryFile import Common.GlobalData as GlobalData from GenFds.GenFds import GenFds, GenFdsApi import multiprocessing as mp from multiprocessing import Manager +from AutoGen.DataPipe import MemoryDataPipe +from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo +from GenFds.FdfParser import FdfParser ## standard targets of build command gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run'] @@ -837,11 +840,11 @@ class Build(): for PcdMa in PcdMaList: if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: PcdMa.GenModuleFilesHash(share_data) PcdMa.GenPreMakefileHash(share_data) if PcdMa.CanSkipbyPreMakefileCache(share_data): - continue + continue PcdMa.CreateCodeFile(False) PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[])) if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: @@ -1218,18 +1221,28 @@ class Build(): mqueue = mp.Queue() for m in AutoGenObject.GetAllModuleInfo: mqueue.put(m) AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target} + AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp} + AutoGenObject.CreateLibModuelDirs() + AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList} + AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList} + AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": AutoGenObject.Workspace.GenFdsCommandDict} self.Progress.Start("Generating makefile and code") data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch)) AutoGenObject.DataPipe.dump(data_pipe_file) autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR) + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") + with open(AutoGenIdFile,"w") as fw: + fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList))) + fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir) + fw.write("PlatformGuid=%s\n" % str(AutoGenObject.Guid)) self.Progress.Stop("done!") if not autogen_rt: self.AutoGenMgr.TerminateWorkers() - self.AutoGenMgr.join(0.1) + self.AutoGenMgr.join(1) raise FatalError(errorcode) AutoGenObject.CreateCodeFile(False) AutoGenObject.CreateMakeFile(False) else: # always recreate top/platform makefile when clean, just in case of inconsistency @@ -1778,10 +1791,11 @@ class Build(): self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList) # # Save MAP buffer into MAP file. # self._SaveMapFile (MapBuffer, Wa) + self.CreateGuidedSectionToolsFile(Wa) ## Build active module for different build targets, different tool chains and different archs # def _BuildModule(self): for BuildTarget in self.BuildTargetList: @@ -1852,13 +1866,13 @@ class Build(): if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: Ma.GenModuleFilesHash(GlobalData.gCacheIR) Ma.GenPreMakefileHash(GlobalData.gCacheIR) if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): - self.HashSkipModules.append(Ma) - EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) - continue + self.HashSkipModules.append(Ma) + EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) + continue # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds' if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile if not self.SkipAutoGen or self.Target == 'genc': @@ -1987,125 +2001,221 @@ class Build(): GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData) for Cmd in GenFfsDict: tmpInf, tmpArch = GenFfsDict[Cmd] CmdSetDict[tmpInf, tmpArch].add(Cmd) return CmdSetDict + def VerifyAutoGenFiles(self): + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") + try: + with open(AutoGenIdFile) as fd: + lines = fd.readlines() + except: + return None + for line in lines: + if "Arch" in line: + ArchList = line.strip().split("=")[1].split("|") + if "BuildDir" in line: + BuildDir = line.split("=")[1].strip() + if "PlatformGuid" in line: + PlatformGuid = line.split("=")[1].strip() + GlobalVarList = [] + for arch in ArchList: + global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch)) + if not os.path.exists(global_var): + return None + GlobalVarList.append(global_var) + for global_var in GlobalVarList: + data_pipe = MemoryDataPipe() + data_pipe.load(global_var) + target = data_pipe.Get("P_Info").get("Target") + toolchain = data_pipe.Get("P_Info").get("ToolChain") + archlist = data_pipe.Get("P_Info").get("ArchList") + Arch = data_pipe.Get("P_Info").get("Arch") + active_p = data_pipe.Get("P_Info").get("ActivePlatform") + workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir") + PackagesPath = os.getenv("PACKAGES_PATH") + mws.setWs(workspacedir, PackagesPath) + LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList") + ModuleBuildDirectoryList = data_pipe.Get("ModuleBuildDirectoryList") + + for m_build_dir in LibraryBuildDirectoryList: + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])): + return None + for m_build_dir in ModuleBuildDirectoryList: + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])): + return None + Wa = WorkSpaceInfo( + workspacedir,active_p,target,toolchain,archlist + ) + Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe) + Wa.AutoGenObjectList.append(Pa) + return Wa + def SetupMakeSetting(self,Wa): + BuildModules = [] + for Pa in Wa.AutoGenObjectList: + for m in Pa._MbList: + ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe) + BuildModules.append(ma) + fdf_file = Wa.FlashDefinition + if fdf_file: + Fdf = FdfParser(fdf_file.Path) + Fdf.ParseFile() + GlobalData.gFdfParser = Fdf + if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict: + FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName] + for FdRegion in FdDict.RegionList: + if str(FdRegion.RegionType) is 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList): + if int(FdRegion.Offset) % 8 != 0: + EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' % (FdRegion.Offset)) + Wa.FdfProfile = Fdf.Profile + self.Fdf = Fdf + else: + self.Fdf = None + return BuildModules ## Build a platform in multi-thread mode # - def _MultiThreadBuildPlatform(self): - SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False) - for BuildTarget in self.BuildTargetList: - GlobalData.gGlobalDefines['TARGET'] = BuildTarget - index = 0 - for ToolChain in self.ToolChainList: - WorkspaceAutoGenTime = time.time() - GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain - GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain - GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index] - index += 1 - Wa = WorkspaceAutoGen( - self.WorkspaceDir, - self.PlatformFile, - BuildTarget, - ToolChain, - self.ArchList, - self.BuildDatabase, - self.TargetTxt, - self.ToolDef, - self.Fdf, - self.FdList, - self.FvList, - self.CapList, - self.SkuId, - self.UniFlag, - self.Progress - ) - self.Fdf = Wa.FdfFile - self.LoadFixAddress = Wa.Platform.LoadFixAddress - self.BuildReport.AddPlatformReport(Wa) - Wa.CreateMakeFile(False) + def PerformAutoGen(self,BuildTarget,ToolChain): + WorkspaceAutoGenTime = time.time() + Wa = WorkspaceAutoGen( + self.WorkspaceDir, + self.PlatformFile, + BuildTarget, + ToolChain, + self.ArchList, + self.BuildDatabase, + self.TargetTxt, + self.ToolDef, + self.Fdf, + self.FdList, + self.FvList, + self.CapList, + self.SkuId, + self.UniFlag, + self.Progress + ) + self.Fdf = Wa.FdfFile + self.LoadFixAddress = Wa.Platform.LoadFixAddress + self.BuildReport.AddPlatformReport(Wa) + Wa.CreateMakeFile(False) # Add ffs build to makefile - CmdListDict = {} - if GlobalData.gEnableGenfdsMultiThread and self.Fdf: - CmdListDict = self._GenFfsCmd(Wa.ArchList) + CmdListDict = {} + if GlobalData.gEnableGenfdsMultiThread and self.Fdf: + CmdListDict = self._GenFfsCmd(Wa.ArchList) - # Add Platform and Package level hash in share_data for module hash calculation later - if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: - GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash - for PkgName in GlobalData.gPackageHash.keys(): - GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName] + # Add Platform and Package level hash in share_data for module hash calculation later + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash + for PkgName in GlobalData.gPackageHash.keys(): + GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName] - # multi-thread exit flag - ExitFlag = threading.Event() - ExitFlag.clear() - self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) - self.BuildModules = [] - TotalModules = [] - for Arch in Wa.ArchList: - PcdMaList = [] - AutoGenStart = time.time() - GlobalData.gGlobalDefines['ARCH'] = Arch - Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) - if Pa is None: - continue - ModuleList = [] - for Inf in Pa.Platform.Modules: - ModuleList.append(Inf) + self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) + BuildModules = [] + TotalModules = [] + for Arch in Wa.ArchList: + PcdMaList = [] + AutoGenStart = time.time() + GlobalData.gGlobalDefines['ARCH'] = Arch + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) + if Pa is None: + continue + ModuleList = [] + for Inf in Pa.Platform.Modules: + ModuleList.append(Inf) # Add the INF only list in FDF - if GlobalData.gFdfParser is not None: - for InfName in GlobalData.gFdfParser.Profile.InfList: - Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch) - if Inf in Pa.Platform.Modules: - continue - ModuleList.append(Inf) - Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} - Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} - Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} - for Module in ModuleList: + if GlobalData.gFdfParser is not None: + for InfName in GlobalData.gFdfParser.Profile.InfList: + Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch) + if Inf in Pa.Platform.Modules: + continue + ModuleList.append(Inf) + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} + Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} + Pa.CreateLibModuelDirs() + Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList} + Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList} + Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict} + ModuleCodaFile = {} + for ma in Pa.ModuleAutoGenList: + ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in ma.CodaTargetList] + Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile} + for Module in ModuleList: # Get ModuleAutoGen object to generate C code file and makefile - Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) + Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) - if Ma is None: + if Ma is None: + continue + if Ma.PcdIsDriver: + Ma.PlatformInfo = Pa + Ma.Workspace = Wa + PcdMaList.append(Ma) + TotalModules.append(Ma) + # Initialize all modules in tracking to 'FAIL' + GlobalData.gModuleBuildTracking[Ma] = 'FAIL' + + + mqueue = mp.Queue() + for m in Pa.GetAllModuleInfo: + mqueue.put(m) + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) + Pa.DataPipe.dump(data_pipe_file) + + autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR) + + # Skip cache hit modules + if GlobalData.gBinCacheSource: + for Ma in TotalModules: + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit: + self.HashSkipModules.append(Ma) + continue + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit: + self.HashSkipModules.append(Ma) continue - if Ma.PcdIsDriver: - Ma.PlatformInfo = Pa - Ma.Workspace = Wa - PcdMaList.append(Ma) - TotalModules.append(Ma) - # Initialize all modules in tracking to 'FAIL' - GlobalData.gModuleBuildTracking[Ma] = 'FAIL' + BuildModules.append(Ma) + else: + BuildModules.extend(TotalModules) - mqueue = mp.Queue() - for m in Pa.GetAllModuleInfo: - mqueue.put(m) - data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) - Pa.DataPipe.dump(data_pipe_file) - autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR) + if not autogen_rt: + self.AutoGenMgr.TerminateWorkers() + self.AutoGenMgr.join(1) + raise FatalError(errorcode) + self.AutoGenTime += int(round((time.time() - AutoGenStart))) + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") + with open(AutoGenIdFile,"w") as fw: + fw.write("Arch=%s\n" % "|".join((Wa.ArchList))) + fw.write("BuildDir=%s\n" % Wa.BuildDir) + fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid)) + self.Progress.Stop("done!") + return Wa, BuildModules - # Skip cache hit modules - if GlobalData.gBinCacheSource: - for Ma in TotalModules: - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit: - self.HashSkipModules.append(Ma) - continue - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit: - self.HashSkipModules.append(Ma) - continue - self.BuildModules.append(Ma) + def _MultiThreadBuildPlatform(self): + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False) + for BuildTarget in self.BuildTargetList: + GlobalData.gGlobalDefines['TARGET'] = BuildTarget + index = 0 + for ToolChain in self.ToolChainList: + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain + GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index] + index += 1 + ExitFlag = threading.Event() + ExitFlag.clear() + if self.SkipAutoGen: + Wa = self.VerifyAutoGenFiles() + if Wa is None: + Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain) else: - self.BuildModules.extend(TotalModules) - - if not autogen_rt: - self.AutoGenMgr.TerminateWorkers() - self.AutoGenMgr.join(0.1) - raise FatalError(errorcode) - self.AutoGenTime += int(round((time.time() - AutoGenStart))) - self.Progress.Stop("done!") + GlobalData.gAutoGenPhase = True + self.BuildModules = self.SetupMakeSetting(Wa) + else: + Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain) + Pa = Wa.AutoGenObjectList[0] + GlobalData.gAutoGenPhase = False if GlobalData.gBinCacheSource: EdkLogger.quiet("Total cache hit driver num: %s, cache miss driver num: %s" % (len(set(self.HashSkipModules)), len(set(self.BuildModules)))) CacheHitMa = set() CacheNotHitMa = set() @@ -2136,18 +2246,17 @@ class Build(): # Start task scheduler if not BuildTask.IsOnGoing(): BuildTask.StartScheduler(self.ThreadNumber, ExitFlag) # in case there's an interruption. we need a full version of makefile for platform - Pa.CreateMakeFile(False) + if BuildTask.HasError(): self.invalidateHash() EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) self.MakeTime += int(round((time.time() - MakeStart))) MakeContiue = time.time() - # # # All modules have been put in build tasks queue. Tell task scheduler # to exit if all tasks are completed # @@ -2175,17 +2284,12 @@ class Build(): if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000: EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules") # # Get Module List # - ModuleList = {} - for Pa in Wa.AutoGenObjectList: - for Ma in Pa.ModuleAutoGenList: - if Ma is None: - continue - if not Ma.IsLibrary: - ModuleList[Ma.Guid.upper()] = Ma + ModuleList = {ma.Guid.upper():ma for ma in self.BuildModules} + # # Rebase module to the preferred memory address before GenFds # MapBuffer = [] if self.LoadFixAddress != 0: @@ -2206,33 +2310,17 @@ class Build(): self.GenFdsTime += int(round((time.time() - GenFdsStart))) # # Save MAP buffer into MAP file. # self._SaveMapFile(MapBuffer, Wa) + self.CreateGuidedSectionToolsFile(Wa) self.invalidateHash() - ## Generate GuidedSectionTools.txt in the FV directories. # - def CreateGuidedSectionToolsFile(self): + def CreateGuidedSectionToolsFile(self,Wa): for BuildTarget in self.BuildTargetList: for ToolChain in self.ToolChainList: - Wa = WorkspaceAutoGen( - self.WorkspaceDir, - self.PlatformFile, - BuildTarget, - ToolChain, - self.ArchList, - self.BuildDatabase, - self.TargetTxt, - self.ToolDef, - self.Fdf, - self.FdList, - self.FvList, - self.CapList, - self.SkuId, - self.UniFlag - ) FvDir = Wa.FvDir if not os.path.exists(FvDir): continue for Arch in self.ArchList: @@ -2285,11 +2373,10 @@ class Build(): if not self.SpawnMode or self.Target not in ["", "all"]: self.SpawnMode = False self._BuildPlatform() else: self._MultiThreadBuildPlatform() - self.CreateGuidedSectionToolsFile() else: self.SpawnMode = False self._BuildModule() if self.Target == 'cleanall': -- 2.20.1.windows.1