From mboxrd@z Thu Jan 1 00:00:00 1970 Authentication-Results: mx.groups.io; dkim=missing; spf=fail (domain: intel.com, ip: , mailfrom: bob.c.feng@intel.com) Received: from mga02.intel.com (mga02.intel.com []) by groups.io with SMTP; Wed, 31 Jul 2019 20:03:48 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga004.jf.intel.com ([10.7.209.38]) by orsmga101.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 31 Jul 2019 20:03:48 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.64,332,1559545200"; d="scan'208";a="324044521" Received: from shwdepsi1121.ccr.corp.intel.com ([10.239.158.47]) by orsmga004.jf.intel.com with ESMTP; 31 Jul 2019 20:03:46 -0700 From: "Bob Feng" To: devel@edk2.groups.io Cc: Liming Gao , Bob Feng Subject: [Patch 05/10 V6] BaseTools: Enable Multiple Process AutoGen Date: Thu, 1 Aug 2019 11:03:28 +0800 Message-Id: <20190801030333.8588-6-bob.c.feng@intel.com> X-Mailer: git-send-email 2.20.1.windows.1 In-Reply-To: <20190801030333.8588-1-bob.c.feng@intel.com> References: <20190801030333.8588-1-bob.c.feng@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1875 Assign the Module AutoGen tasks into multiple sub process. Cc: Liming Gao Signed-off-by: Bob Feng --- .../Source/Python/AutoGen/AutoGenWorker.py | 192 ++++++++++++++++++ BaseTools/Source/Python/AutoGen/DataPipe.py | 17 +- BaseTools/Source/Python/AutoGen/GenC.py | 4 +- .../Source/Python/AutoGen/ModuleAutoGen.py | 7 +- .../Source/Python/AutoGen/PlatformAutoGen.py | 10 +- .../Source/Python/AutoGen/WorkspaceAutoGen.py | 2 + BaseTools/Source/Python/build/build.py | 124 +++++++---- 7 files changed, 300 insertions(+), 56 deletions(-) create mode 100644 BaseTools/Source/Python/AutoGen/AutoGenWorker.py diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py new file mode 100644 index 000000000000..8736fde50ef7 --- /dev/null +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -0,0 +1,192 @@ +## @file +# Create makefile for MS nmake and GNU make +# +# Copyright (c) 2019, Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +# +from __future__ import absolute_import +import multiprocessing as mp +import threading +from Common.Misc import PathClass +from AutoGen.ModuleAutoGen import ModuleAutoGen +from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo +import Common.GlobalData as GlobalData +import Common.EdkLogger as EdkLogger +import os +from Common.MultipleWorkspace import MultipleWorkspace as mws +from AutoGen.AutoGen import AutoGen +from Workspace.WorkspaceDatabase import BuildDB +from queue import Empty +import traceback +import sys +from AutoGen.DataPipe import MemoryDataPipe +def clearQ(q): + try: + while True: + q.get_nowait() + except Empty: + pass +class AutoGenManager(threading.Thread): + def __init__(self,autogen_workers, feedback_q,error_event): + super(AutoGenManager,self).__init__() + self.autogen_workers = autogen_workers + self.feedback_q = feedback_q + self.terminate = False + self.Status = True + self.error_event = error_event + def run(self): + try: + fin_num = 0 + while True: + badnews = self.feedback_q.get() + if badnews is None: + break + if badnews == "Done": + fin_num += 1 + else: + self.Status = False + self.TerminateWorkers() + if fin_num == len(self.autogen_workers): + self.clearQueue() + for w in self.autogen_workers: + w.join() + break + except Exception: + return + + def clearQueue(self): + taskq = self.autogen_workers[0].module_queue + clearQ(taskq) + clearQ(self.feedback_q) + + def TerminateWorkers(self): + self.error_event.set() + def kill(self): + self.feedback_q.put(None) +class AutoGenWorkerInProcess(mp.Process): + def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,error_event): + mp.Process.__init__(self) + self.module_queue = module_queue + self.data_pipe_file_path =data_pipe_file_path + self.data_pipe = None + self.feedback_q = feedback_q + self.PlatformMetaFileSet = {} + self.file_lock = file_lock + self.error_event = error_event + def GetPlatformMetaFile(self,filepath,root): + try: + return self.PlatformMetaFileSet[(filepath,root)] + except: + self.PlatformMetaFileSet[(filepath,root)] = filepath + return self.PlatformMetaFileSet[(filepath,root)] + def run(self): + try: + taskname = "Init" + with self.file_lock: + if not os.path.exists(self.data_pipe_file_path): + self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path) + self.data_pipe = MemoryDataPipe() + self.data_pipe.load(self.data_pipe_file_path) + EdkLogger.Initialize() + loglevel = self.data_pipe.Get("LogLevel") + if not loglevel: + loglevel = EdkLogger.INFO + EdkLogger.SetLevel(loglevel) + logfile = self.data_pipe.Get("LogFile") + if logfile: + EdkLogger.SetLogFile(logfile) + target = self.data_pipe.Get("P_Info").get("Target") + toolchain = self.data_pipe.Get("P_Info").get("ToolChain") + archlist = self.data_pipe.Get("P_Info").get("ArchList") + + active_p = self.data_pipe.Get("P_Info").get("ActivePlatform") + workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir") + PackagesPath = os.getenv("PACKAGES_PATH") + mws.setWs(workspacedir, PackagesPath) + self.Wa = WorkSpaceInfo( + workspacedir,active_p,target,toolchain,archlist + ) + self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp") + GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines") + GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines") + os.environ._data = self.data_pipe.Get("Env_Var") + GlobalData.gWorkspace = workspacedir + GlobalData.gDisableIncludePathCheck = False + GlobalData.gFdfParser = self.data_pipe.Get("FdfParser") + GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath") + pcd_from_build_option = [] + for pcd_tuple in self.data_pipe.Get("BuildOptPcd"): + pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1])) + if pcd_tuple[2].strip(): + pcd_id = ".".join((pcd_id,pcd_tuple[2])) + pcd_from_build_option.append("=".join((pcd_id,pcd_tuple[3]))) + GlobalData.BuildOptionPcd = pcd_from_build_option + module_count = 0 + FfsCmd = self.data_pipe.Get("FfsCommand") + if FfsCmd is None: + FfsCmd = {} + PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"), + self.data_pipe.Get("P_Info").get("WorkspaceDir")) + libConstPcd = self.data_pipe.Get("LibConstPcd") + Refes = self.data_pipe.Get("REFS") + while True: + if self.module_queue.empty(): + break + if self.error_event.is_set(): + break + module_count += 1 + module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait() + modulefullpath = os.path.join(module_root,module_file) + taskname = " : ".join((modulefullpath,module_arch)) + module_metafile = PathClass(module_file,module_root) + if module_path: + module_metafile.Path = module_path + if module_basename: + module_metafile.BaseName = module_basename + if module_originalpath: + module_metafile.OriginalPath = PathClass(module_originalpath,module_root) + arch = module_arch + target = self.data_pipe.Get("P_Info").get("Target") + toolchain = self.data_pipe.Get("P_Info").get("ToolChain") + Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe) + Ma.IsLibrary = IsLib + if IsLib: + if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd: + Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] + if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes: + Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] + Ma.CreateCodeFile() + Ma.CreateMakeFile(GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[])) + Ma.CreateAsBuiltInf() + except Empty: + pass + except: + traceback.print_exc(file=sys.stdout) + self.feedback_q.put(taskname) + finally: + self.feedback_q.put("Done") + def printStatus(self): + print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache()))) + print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache()))) + groupobj = {} + for buildobj in BuildDB.BuildObject.GetCache().values(): + if str(buildobj).lower().endswith("dec"): + try: + groupobj['dec'].append(str(buildobj)) + except: + groupobj['dec'] = [str(buildobj)] + if str(buildobj).lower().endswith("dsc"): + try: + groupobj['dsc'].append(str(buildobj)) + except: + groupobj['dsc'] = [str(buildobj)] + + if str(buildobj).lower().endswith("inf"): + try: + groupobj['inf'].append(str(buildobj)) + except: + groupobj['inf'] = [str(buildobj)] + + print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(groupobj.get("dec",[])))) + print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(groupobj.get("dsc",[])))) + print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(groupobj.get("inf",[])))) diff --git a/BaseTools/Source/Python/AutoGen/DataPipe.py b/BaseTools/Source/Python/AutoGen/DataPipe.py index 5bcc39bd380d..62992080567f 100644 --- a/BaseTools/Source/Python/AutoGen/DataPipe.py +++ b/BaseTools/Source/Python/AutoGen/DataPipe.py @@ -9,10 +9,11 @@ from Workspace.WorkspaceDatabase import BuildDB from Workspace.WorkspaceCommon import GetModuleLibInstances import Common.GlobalData as GlobalData import os import pickle from pickle import HIGHEST_PROTOCOL +from Common import EdkLogger class PCD_DATA(): def __init__(self,TokenCName,TokenSpaceGuidCName,Type,DatumType,SkuInfoList,DefaultValue, MaxDatumSize,UserDefinedDefaultStoresFlag,validateranges, validlists,expressions,CustomAttribute,TokenValue): @@ -32,17 +33,19 @@ class PCD_DATA(): class DataPipe(object): def __init__(self, BuildDir=None): self.data_container = {} self.BuildDir = BuildDir + self.dump_file = "" class MemoryDataPipe(DataPipe): def Get(self,key): return self.data_container.get(key) def dump(self,file_path): + self.dump_file = file_path with open(file_path,'wb') as fd: pickle.dump(self.data_container,fd,pickle.HIGHEST_PROTOCOL) def load(self,file_path): with open(file_path,'rb') as fd: @@ -69,11 +72,11 @@ class MemoryDataPipe(DataPipe): #Platform Module Pcds ModulePcds = {} for m in PlatformInfo.Platform.Modules: m_pcds = PlatformInfo.Platform.Modules[m].Pcds if m_pcds: - ModulePcds[(m.File,m.Root)] = [PCD_DATA( + ModulePcds[(m.File,m.Root,m.Arch)] = [PCD_DATA( pcd.TokenCName,pcd.TokenSpaceGuidCName,pcd.Type, pcd.DatumType,pcd.SkuInfoList,pcd.DefaultValue, pcd.MaxDatumSize,pcd.UserDefinedDefaultStoresFlag,pcd.validateranges, pcd.validlists,pcd.expressions,pcd.CustomAttribute,pcd.TokenValue) for pcd in PlatformInfo.Platform.Modules[m].Pcds.values()] @@ -81,15 +84,22 @@ class MemoryDataPipe(DataPipe): self.DataContainer = {"MOL_PCDS":ModulePcds} #Module's Library Instance ModuleLibs = {} + libModules = {} for m in PlatformInfo.Platform.Modules: module_obj = BuildDB.BuildObject[m,PlatformInfo.Arch,PlatformInfo.BuildTarget,PlatformInfo.ToolChain] Libs = GetModuleLibInstances(module_obj, PlatformInfo.Platform, BuildDB.BuildObject, PlatformInfo.Arch,PlatformInfo.BuildTarget,PlatformInfo.ToolChain) - ModuleLibs[(m.File,m.Root,module_obj.Arch)] = [(l.MetaFile.File,l.MetaFile.Root,l.Arch) for l in Libs] + for lib in Libs: + try: + libModules[(lib.MetaFile.File,lib.MetaFile.Root,lib.Arch,lib.MetaFile.Path)].append((m.File,m.Root,module_obj.Arch,m.Path)) + except: + libModules[(lib.MetaFile.File,lib.MetaFile.Root,lib.Arch,lib.MetaFile.Path)] = [(m.File,m.Root,module_obj.Arch,m.Path)] + ModuleLibs[(m.File,m.Root,module_obj.Arch,m.Path)] = [(l.MetaFile.File,l.MetaFile.Root,l.Arch,l.MetaFile.Path) for l in Libs] self.DataContainer = {"DEPS":ModuleLibs} + self.DataContainer = {"REFS":libModules} #Platform BuildOptions platform_build_opt = PlatformInfo.EdkIIBuildOption @@ -141,7 +151,10 @@ class MemoryDataPipe(DataPipe): self.DataContainer = {"PackageList": [(dec.MetaFile,dec.Arch) for dec in PlatformInfo.PackageList]} self.DataContainer = {"GuidDict": PlatformInfo.Platform._GuidDict} + self.DataContainer = {"DatabasePath":GlobalData.gDatabasePath} self.DataContainer = {"FdfParser": True if GlobalData.gFdfParser else False} + self.DataContainer = {"LogLevel": EdkLogger.GetLevel()} + self.DataContainer = {"LogFile": GlobalData.gOptions.LogFile if GlobalData.gOptions.LogFile is not None else ""} diff --git a/BaseTools/Source/Python/AutoGen/GenC.py b/BaseTools/Source/Python/AutoGen/GenC.py index 4c3f4e3e55ae..910c8fe3706c 100644 --- a/BaseTools/Source/Python/AutoGen/GenC.py +++ b/BaseTools/Source/Python/AutoGen/GenC.py @@ -1470,12 +1470,12 @@ def CreateModuleEntryPointCode(Info, AutoGenC, AutoGenH): 'UefiSpecVersion': UefiSpecVersion + 'U' } if Info.ModuleType in [SUP_MODULE_PEI_CORE, SUP_MODULE_DXE_CORE, SUP_MODULE_SMM_CORE, SUP_MODULE_MM_CORE_STANDALONE]: if Info.SourceFileList: - if NumEntryPoints != 1: - EdkLogger.error( + if NumEntryPoints != 1: + EdkLogger.error( "build", AUTOGEN_ERROR, '%s must have exactly one entry point' % Info.ModuleType, File=str(Info), ExtraData= ", ".join(Info.Module.ModuleEntryPointList) diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py index d19c03862094..ab8ef6895585 100644 --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py @@ -1681,13 +1681,11 @@ class ModuleAutoGen(AutoGen): if self.IsBinaryModule: return self.GenFfsList = GenFfsList - if not self.IsLibrary and CreateLibraryMakeFile: - for LibraryAutoGen in self.LibraryAutoGenList: - LibraryAutoGen.CreateMakeFile() + # Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping if not GlobalData.gUseHashCache and self.CanSkip(): return if len(self.CustomMakefile) == 0: @@ -1724,13 +1722,10 @@ class ModuleAutoGen(AutoGen): if self.IsBinaryModule: if self.IsLibrary: self.CopyBinaryFiles() return - if not self.IsLibrary and CreateLibraryCodeFile: - for LibraryAutoGen in self.LibraryAutoGenList: - LibraryAutoGen.CreateCodeFile() # Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping if not GlobalData.gUseHashCache and self.CanSkip(): return diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py index 6c947eca2b57..4cb9966415b6 100644 --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py @@ -131,10 +131,16 @@ class PlatformAutoGen(AutoGen): self.DataPipe = MemoryDataPipe(self.BuildDir) self.DataPipe.FillData(self) return True + def FillData_LibConstPcd(self): + libConstPcd = {} + for LibAuto in self.LibraryAutoGenList: + if LibAuto.ConstPcd: + libConstPcd[(LibAuto.MetaFile.File,LibAuto.MetaFile.Root,LibAuto.Arch,LibAuto.MetaFile.Path)] = LibAuto.ConstPcd + self.DataPipe.DataContainer = {"LibConstPcd":libConstPcd} ## hash() operator of PlatformAutoGen # # The platform file path and arch string will be used to represent # hash value of this object # @@ -1084,14 +1090,14 @@ class PlatformAutoGen(AutoGen): module_obj = self.BuildDatabase[m,self.Arch,self.BuildTarget,self.ToolChain] if not bool(module_obj.LibraryClass): Libs = GetModuleLibInstances(module_obj, self.Platform, self.BuildDatabase, self.Arch,self.BuildTarget,self.ToolChain) else: Libs = [] - ModuleLibs.update( set([(l.MetaFile.File,l.MetaFile.Root,l.Arch,True) for l in Libs])) + ModuleLibs.update( set([(l.MetaFile.File,l.MetaFile.Root,l.MetaFile.Path,l.MetaFile.BaseName,l.MetaFile.OriginalPath,l.Arch,True) for l in Libs])) if WithoutPcd and module_obj.PcdIsDriver: continue - ModuleLibs.add((m.File,m.Root,module_obj.Arch,False)) + ModuleLibs.add((m.File,m.Root,m.Path,m.BaseName,m.OriginalPath,module_obj.Arch,bool(module_obj.LibraryClass))) return ModuleLibs ## Resolve the library classes in a module to library instances # diff --git a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py index ab58b21772c3..4ad92653a238 100644 --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py @@ -111,10 +111,12 @@ class WorkspaceAutoGen(AutoGen): self.ProcessModuleFromPdf() self.ProcessPcdType() self.ProcessMixedPcd() self.VerifyPcdsFromFDF() self.CollectAllPcds() + for Pa in self.AutoGenObjectList: + Pa.FillData_LibConstPcd() self.GeneratePkgLevelHash() # # Check PCDs token value conflict in each DEC file. # self._CheckAllPcdsTokenValueConflict() diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index 3d083f4eaade..579b57a4e958 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -28,10 +28,11 @@ from subprocess import Popen,PIPE from collections import OrderedDict, defaultdict from optparse import OptionParser from AutoGen.PlatformAutoGen import PlatformAutoGen from AutoGen.ModuleAutoGen import ModuleAutoGen from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen +from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager from AutoGen import GenMake from Common import Misc as Utils from Common.TargetTxtClassObject import TargetTxt from Common.ToolDefClassObject import ToolDef @@ -48,11 +49,11 @@ from BuildReport import BuildReport from GenPatchPcdTable.GenPatchPcdTable import PeImageClass,parsePcdInfoFromMapFile from PatchPcdValue.PatchPcdValue import PatchBinaryFile import Common.GlobalData as GlobalData from GenFds.GenFds import GenFds, GenFdsApi - +import multiprocessing as mp # Version and Copyright VersionNumber = "0.60" + ' ' + gBUILD_VERSION __version__ = "%prog Version " + VersionNumber __copyright__ = "Copyright (c) 2007 - 2018, Intel Corporation All rights reserved." @@ -341,13 +342,13 @@ class ModuleMakeUnit(BuildUnit): # # @param self The object pointer # @param Obj The ModuleAutoGen object the build is working on # @param Target The build target name, one of gSupportedTarget # - def __init__(self, Obj, Target): - Dependency = [ModuleMakeUnit(La, Target) for La in Obj.LibraryAutoGenList] - BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir) + def __init__(self, Obj, BuildCommand,Target): + Dependency = [ModuleMakeUnit(La, BuildCommand,Target) for La in Obj.LibraryAutoGenList] + BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir) if Target in [None, "", "all"]: self.Target = "tbuild" ## The smallest platform unit that can be built by nmake/make command in multi-thread build mode # @@ -362,14 +363,14 @@ class PlatformMakeUnit(BuildUnit): # # @param self The object pointer # @param Obj The PlatformAutoGen object the build is working on # @param Target The build target name, one of gSupportedTarget # - def __init__(self, Obj, Target): - Dependency = [ModuleMakeUnit(Lib, Target) for Lib in self.BuildObject.LibraryAutoGenList] - Dependency.extend([ModuleMakeUnit(Mod, Target) for Mod in self.BuildObject.ModuleAutoGenList]) - BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir) + def __init__(self, Obj, BuildCommand, Target): + Dependency = [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib in self.BuildObject.LibraryAutoGenList] + Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mod in self.BuildObject.ModuleAutoGenList]) + BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir) ## The class representing the task of a module build or platform build # # This class manages the build tasks in multi-thread build mode. Its jobs include # scheduling thread running, catching thread error, monitor the thread status, etc. @@ -822,12 +823,36 @@ class Build(): self.TargetTxt = TargetTxt self.ToolDef = ToolDef if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)): self.InitBuild() + self.AutoGenMgr = None EdkLogger.info("") os.chdir(self.WorkspaceDir) + def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList): + try: + if SkipAutoGen: + return True,0 + feedback_q = mp.Queue() + file_lock = mp.Lock() + error_event = mp.Event() + auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock,error_event) for _ in range(self.ThreadNumber)] + self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event) + self.AutoGenMgr.start() + for w in auto_workers: + w.start() + if PcdMaList is not None: + for PcdMa in PcdMaList: + PcdMa.CreateCodeFile(True) + PcdMa.CreateMakeFile(GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[])) + PcdMa.CreateAsBuiltInf() + + self.AutoGenMgr.join() + rt = self.AutoGenMgr.Status + return rt, 0 + except Exception as e: + return False,e.errcode ## Load configuration # # This method will parse target.txt and get the build configurations. # @@ -1188,30 +1213,33 @@ class Build(): # @param CreateDepModuleCodeFile Flag used to indicate creating code # for dependent modules/Libraries # @param CreateDepModuleMakeFile Flag used to indicate creating makefile # for dependent modules/Libraries # - def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand={}): + def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand=None, PcdMaList=None): if AutoGenObject is None: return False - + if FfsCommand is None: + FfsCommand = {} # skip file generation for cleanxxx targets, run and fds target if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile - if not self.SkipAutoGen or Target == 'genc': - self.Progress.Start("Generating code") - AutoGenObject.CreateCodeFile(CreateDepsCodeFile) - self.Progress.Stop("done!") - if Target == "genc": - return True + mqueue = mp.Queue() + for m in AutoGenObject.GetAllModuleInfo: + mqueue.put(m) - if not self.SkipAutoGen or Target == 'genmake': - self.Progress.Start("Generating makefile") - AutoGenObject.CreateMakeFile(CreateDepsMakeFile, FfsCommand) - self.Progress.Stop("done!") - if Target == "genmake": - return True + AutoGenObject.DataPipe.DataContainer = {"FfsCommand":FfsCommand} + self.Progress.Start("Generating makefile and code") + data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch)) + AutoGenObject.DataPipe.dump(data_pipe_file) + autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList) + self.Progress.Stop("done!") + if not autogen_rt: + self.AutoGenMgr.TerminateWorkers() + self.AutoGenMgr.join(0.1) + raise FatalError(errorcode) + return autogen_rt else: # always recreate top/platform makefile when clean, just in case of inconsistency AutoGenObject.CreateCodeFile(False) AutoGenObject.CreateMakeFile(False) @@ -1234,11 +1262,10 @@ class Build(): # build modules if BuildModule: BuildCommand = BuildCommand + [Target] LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir) - self.CreateAsBuiltInf() if GlobalData.gBinCacheDest: self.UpdateBuildCache() self.BuildModules = [] return True @@ -1255,11 +1282,10 @@ class Build(): NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild'] LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir) for Mod in AutoGenObject.ModuleBuildDirectoryList: NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, makefile)), 'pbuild'] LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir) - self.CreateAsBuiltInf() if GlobalData.gBinCacheDest: self.UpdateBuildCache() self.BuildModules = [] return True @@ -1711,13 +1737,14 @@ class Build(): Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) if Ma is None: continue if Ma.PcdIsDriver: Ma.PlatformInfo = Pa + Ma.Workspace = Wa PcdMaList.append(Ma) self.BuildModules.append(Ma) - self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict) + self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList) # Create MAP file when Load Fix Address is enabled. if self.Target in ["", "all", "fds"]: for Arch in Wa.ArchList: GlobalData.gGlobalDefines['ARCH'] = Arch @@ -1848,11 +1875,11 @@ class Build(): GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL' self.AutoGenTime += int(round((time.time() - AutoGenStart))) MakeStart = time.time() for Ma in self.BuildModules: if not Ma.IsBinaryModule: - Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target)) + Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target)) # Break build if any build thread has error if BuildTask.HasError(): # we need a full version of makefile for platform ExitFlag.set() BuildTask.WaitForComplete() @@ -1978,18 +2005,19 @@ class Build(): self.LoadFixAddress = Wa.Platform.LoadFixAddress self.BuildReport.AddPlatformReport(Wa) Wa.CreateMakeFile(False) # Add ffs build to makefile - CmdListDict = None + CmdListDict = {} if GlobalData.gEnableGenfdsMultiThread and self.Fdf: CmdListDict = self._GenFfsCmd(Wa.ArchList) # multi-thread exit flag ExitFlag = threading.Event() ExitFlag.clear() self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) + BuildModules = [] for Arch in Wa.ArchList: PcdMaList = [] AutoGenStart = time.time() GlobalData.gGlobalDefines['ARCH'] = Arch Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) @@ -2003,18 +2031,21 @@ class Build(): for InfName in GlobalData.gFdfParser.Profile.InfList: Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch) if Inf in Pa.Platform.Modules: continue ModuleList.append(Inf) + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} + Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} for Module in ModuleList: # Get ModuleAutoGen object to generate C code file and makefile Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) if Ma is None: continue if Ma.PcdIsDriver: Ma.PlatformInfo = Pa + Ma.Workspace = Wa PcdMaList.append(Ma) if Ma.CanSkipbyHash(): self.HashSkipModules.append(Ma) if GlobalData.gBinCacheSource: EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) @@ -2022,38 +2053,36 @@ class Build(): else: if GlobalData.gBinCacheSource: EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds' - if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile - if not self.SkipAutoGen or self.Target == 'genc': - Ma.CreateCodeFile(True) - if self.Target == "genc": - continue - if not self.SkipAutoGen or self.Target == 'genmake': - if CmdListDict and self.Fdf and (Module.File, Arch) in CmdListDict: - Ma.CreateMakeFile(True, CmdListDict[Module.File, Arch]) - del CmdListDict[Module.File, Arch] - else: - Ma.CreateMakeFile(True) - if self.Target == "genmake": - continue - self.BuildModules.append(Ma) + BuildModules.append(Ma) # Initialize all modules in tracking to 'FAIL' if Ma.Arch not in GlobalData.gModuleBuildTracking: GlobalData.gModuleBuildTracking[Ma.Arch] = dict() if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]: GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL' + mqueue = mp.Queue() + for m in Pa.GetAllModuleInfo: + mqueue.put(m) + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) + Pa.DataPipe.dump(data_pipe_file) + autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList) self.Progress.Stop("done!") self.AutoGenTime += int(round((time.time() - AutoGenStart))) + if not autogen_rt: + self.AutoGenMgr.TerminateWorkers() + self.AutoGenMgr.join(0.1) + raise FatalError(errorcode) + for Arch in Wa.ArchList: MakeStart = time.time() - for Ma in self.BuildModules: + for Ma in BuildModules: # Generate build task for the module if not Ma.IsBinaryModule: - Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target)) + Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target)) # Break build if any build thread has error if BuildTask.HasError(): # we need a full version of makefile for platform ExitFlag.set() BuildTask.WaitForComplete() @@ -2078,11 +2107,10 @@ class Build(): # All modules have been put in build tasks queue. Tell task scheduler # to exit if all tasks are completed # ExitFlag.set() BuildTask.WaitForComplete() - self.CreateAsBuiltInf() if GlobalData.gBinCacheDest: self.UpdateBuildCache() self.BuildModules = [] self.MakeTime += int(round((time.time() - MakeContiue))) # @@ -2496,10 +2524,14 @@ def Main(): EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc()) else: EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False) ReturnCode = FORMAT_INVALID except KeyboardInterrupt: + if MyBuild is not None: + + # for multi-thread build exits safely + MyBuild.Relinquish() ReturnCode = ABORT_ERROR if Option is not None and Option.debug is not None: EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc()) except: if MyBuild is not None: @@ -2552,10 +2584,14 @@ def Main(): EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d %Y", time.localtime())) EdkLogger.quiet("Build total time: %s\n" % BuildDurationStr) return ReturnCode if __name__ == '__main__': + try: + mp.set_start_method('spawn') + except: + pass r = Main() ## 0-127 is a safe return range, and 1 is a standard default error if r < 0 or r > 127: r = 1 sys.exit(r) -- 2.20.1.windows.1