From mboxrd@z Thu Jan 1 00:00:00 1970 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 134.134.136.65, mailfrom: bob.c.feng@intel.com) Received: from mga03.intel.com (mga03.intel.com [134.134.136.65]) by groups.io with SMTP; Mon, 22 Jul 2019 01:55:47 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga004.jf.intel.com ([10.7.209.38]) by orsmga103.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 22 Jul 2019 01:51:50 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.64,294,1559545200"; d="scan'208";a="320624186" Received: from shwdepsi1121.ccr.corp.intel.com ([10.239.158.47]) by orsmga004.jf.intel.com with ESMTP; 22 Jul 2019 01:51:49 -0700 From: "Bob Feng" To: devel@edk2.groups.io Cc: Liming Gao , Bob Feng Subject: [Patch 5/9 V2] BaseTools: Enable Multiple Process AutoGen Date: Mon, 22 Jul 2019 16:50:56 +0800 Message-Id: <20190722085100.20552-6-bob.c.feng@intel.com> X-Mailer: git-send-email 2.20.1.windows.1 In-Reply-To: <20190722085100.20552-1-bob.c.feng@intel.com> References: <20190722085100.20552-1-bob.c.feng@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1875 Assign the Module AutoGen tasks into multiple sub process. Cc: Liming Gao Signed-off-by: Bob Feng --- .../Source/Python/AutoGen/AutoGenWorker.py | 161 ++++++++++++++++++ BaseTools/Source/Python/AutoGen/DataPipe.py | 6 + BaseTools/Source/Python/AutoGen/GenC.py | 4 +- .../Source/Python/AutoGen/ModuleAutoGen.py | 7 +- .../Source/Python/AutoGen/PlatformAutoGen.py | 4 +- BaseTools/Source/Python/build/build.py | 112 +++++++----- 6 files changed, 240 insertions(+), 54 deletions(-) create mode 100644 BaseTools/Source/Python/AutoGen/AutoGenWorker.py diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py new file mode 100644 index 000000000000..bb6545288ffc --- /dev/null +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -0,0 +1,161 @@ +## @file +# Create makefile for MS nmake and GNU make +# +# Copyright (c) 2019, Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +# +from __future__ import absolute_import +import multiprocessing as mp +import threading +from Common.Misc import PathClass +from AutoGen.ModuleAutoGen import ModuleAutoGen +from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo +import Common.GlobalData as GlobalData +import Common.EdkLogger as EdkLogger +import os +from Common.MultipleWorkspace import MultipleWorkspace as mws +from AutoGen.AutoGen import AutoGen +from Workspace.WorkspaceDatabase import BuildDB +import time +from queue import Empty +import traceback +import sys +from AutoGen.DataPipe import MemoryDataPipe +class AutoGenManager(threading.Thread): + def __init__(self,autogen_workers, feedback_q): + super(AutoGenManager,self).__init__() + self.autogen_workers = autogen_workers + self.feedback_q = feedback_q + self.terminate = False + self.Status = True + def run(self): + try: + while True: + if self.terminate: + break + if self.feedback_q.empty(): + time.sleep(1) + continue + badnews = self.feedback_q.get(False) + if badnews: + print(badnews) + self.Status = False + self.TerminateWorkers() + break + except Exception: + return + + def kill(self): + self.terminate = True + + def TerminateWorkers(self): + for w in self.autogen_workers: + if w.is_alive(): + w.terminate() + +class AutoGenWorkerInProcess(mp.Process): + def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock): + mp.Process.__init__(self) + self.module_queue = module_queue + self.data_pipe_file_path =data_pipe_file_path + self.data_pipe = None + self.feedback_q = feedback_q + self.PlatformMetaFileSet = {} + self.file_lock = file_lock + def GetPlatformMetaFile(self,filepath,root): + try: + return self.PlatformMetaFileSet[(filepath,root)] + except: + self.PlatformMetaFileSet[(filepath,root)] = filepath + return self.PlatformMetaFileSet[(filepath,root)] + def run(self): + try: + taskname = "Init" + with self.file_lock: + if not os.path.exists(self.data_pipe_file_path): + self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path) + self.data_pipe = MemoryDataPipe() + self.data_pipe.load(self.data_pipe_file_path) + EdkLogger.Initialize() + loglevel = self.data_pipe.Get("LogLevel") + if not loglevel: + loglevel = EdkLogger.INFO + EdkLogger.SetLevel(loglevel) + logfile = self.data_pipe.Get("LogFile") + if logfile: + EdkLogger.SetLogFile(logfile) + target = self.data_pipe.Get("P_Info").get("Target") + toolchain = self.data_pipe.Get("P_Info").get("ToolChain") + archlist = self.data_pipe.Get("P_Info").get("ArchList") + + active_p = self.data_pipe.Get("P_Info").get("ActivePlatform") + workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir") + PackagesPath = os.getenv("PACKAGES_PATH") + mws.setWs(workspacedir, PackagesPath) + self.Wa = WorkSpaceInfo( + workspacedir,active_p,target,toolchain,archlist + ) + GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines") + GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines") + os.environ._data = self.data_pipe.Get("Env_Var") + GlobalData.gWorkspace = workspacedir + GlobalData.gDisableIncludePathCheck = False + GlobalData.gFdfParser = self.data_pipe.Get("FdfParser") + GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath") + module_count = 0 + FfsCmd = self.data_pipe.Get("FfsCommand") + if FfsCmd is None: + FfsCmd = {} + PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"), + self.data_pipe.Get("P_Info").get("WorkspaceDir")) + while not self.module_queue.empty(): + module_count += 1 + module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get() + modulefullpath = os.path.join(module_root,module_file) + taskname = " : ".join((modulefullpath,module_arch)) + module_metafile = PathClass(module_file,module_root) + if module_path: + module_metafile.Path = module_path + if module_basename: + module_metafile.BaseName = module_basename + if module_originalpath: + module_metafile.OriginalPath = PathClass(module_originalpath,module_root) + arch = module_arch + target = self.data_pipe.Get("P_Info").get("Target") + toolchain = self.data_pipe.Get("P_Info").get("ToolChain") + Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe) + Ma.IsLibrary = IsLib + Ma.CreateCodeFile() + Ma.CreateMakeFile(GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[])) + Ma.CreateAsBuiltInf() + except Empty: + pass + except: + traceback.print_exc(file=sys.stdout) + self.feedback_q.put(taskname) + + def printStatus(self): + print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache()))) + print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache()))) + groupobj = {} + for buildobj in BuildDB.BuildObject.GetCache().values(): + if str(buildobj).lower().endswith("dec"): + try: + groupobj['dec'].append(str(buildobj)) + except: + groupobj['dec'] = [str(buildobj)] + if str(buildobj).lower().endswith("dsc"): + try: + groupobj['dsc'].append(str(buildobj)) + except: + groupobj['dsc'] = [str(buildobj)] + + if str(buildobj).lower().endswith("inf"): + try: + groupobj['inf'].append(str(buildobj)) + except: + groupobj['inf'] = [str(buildobj)] + + print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(groupobj.get("dec",[])))) + print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(groupobj.get("dsc",[])))) + print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(groupobj.get("inf",[])))) diff --git a/BaseTools/Source/Python/AutoGen/DataPipe.py b/BaseTools/Source/Python/AutoGen/DataPipe.py index 5bcc39bd380d..9478f41d481b 100644 --- a/BaseTools/Source/Python/AutoGen/DataPipe.py +++ b/BaseTools/Source/Python/AutoGen/DataPipe.py @@ -9,10 +9,11 @@ from Workspace.WorkspaceDatabase import BuildDB from Workspace.WorkspaceCommon import GetModuleLibInstances import Common.GlobalData as GlobalData import os import pickle from pickle import HIGHEST_PROTOCOL +from Common import EdkLogger class PCD_DATA(): def __init__(self,TokenCName,TokenSpaceGuidCName,Type,DatumType,SkuInfoList,DefaultValue, MaxDatumSize,UserDefinedDefaultStoresFlag,validateranges, validlists,expressions,CustomAttribute,TokenValue): @@ -32,17 +33,19 @@ class PCD_DATA(): class DataPipe(object): def __init__(self, BuildDir=None): self.data_container = {} self.BuildDir = BuildDir + self.dump_file = "" class MemoryDataPipe(DataPipe): def Get(self,key): return self.data_container.get(key) def dump(self,file_path): + self.dump_file = file_path with open(file_path,'wb') as fd: pickle.dump(self.data_container,fd,pickle.HIGHEST_PROTOCOL) def load(self,file_path): with open(file_path,'rb') as fd: @@ -141,7 +144,10 @@ class MemoryDataPipe(DataPipe): self.DataContainer = {"PackageList": [(dec.MetaFile,dec.Arch) for dec in PlatformInfo.PackageList]} self.DataContainer = {"GuidDict": PlatformInfo.Platform._GuidDict} + self.DataContainer = {"DatabasePath":GlobalData.gDatabasePath} self.DataContainer = {"FdfParser": True if GlobalData.gFdfParser else False} + self.DataContainer = {"LogLevel": EdkLogger.GetLevel()} + self.DataContainer = {"LogFile": GlobalData.gOptions.LogFile if GlobalData.gOptions.LogFile is not None else ""} diff --git a/BaseTools/Source/Python/AutoGen/GenC.py b/BaseTools/Source/Python/AutoGen/GenC.py index 4c3f4e3e55ae..910c8fe3706c 100644 --- a/BaseTools/Source/Python/AutoGen/GenC.py +++ b/BaseTools/Source/Python/AutoGen/GenC.py @@ -1470,12 +1470,12 @@ def CreateModuleEntryPointCode(Info, AutoGenC, AutoGenH): 'UefiSpecVersion': UefiSpecVersion + 'U' } if Info.ModuleType in [SUP_MODULE_PEI_CORE, SUP_MODULE_DXE_CORE, SUP_MODULE_SMM_CORE, SUP_MODULE_MM_CORE_STANDALONE]: if Info.SourceFileList: - if NumEntryPoints != 1: - EdkLogger.error( + if NumEntryPoints != 1: + EdkLogger.error( "build", AUTOGEN_ERROR, '%s must have exactly one entry point' % Info.ModuleType, File=str(Info), ExtraData= ", ".join(Info.Module.ModuleEntryPointList) diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py index f0a4afc3a664..36bbaffa56d3 100644 --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py @@ -1681,13 +1681,11 @@ class ModuleAutoGen(AutoGen): if self.IsBinaryModule: return self.GenFfsList = GenFfsList - if not self.IsLibrary and CreateLibraryMakeFile: - for LibraryAutoGen in self.LibraryAutoGenList: - LibraryAutoGen.CreateMakeFile() + # Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping if not GlobalData.gUseHashCache and self.CanSkip(): return if len(self.CustomMakefile) == 0: @@ -1724,13 +1722,10 @@ class ModuleAutoGen(AutoGen): if self.IsBinaryModule: if self.IsLibrary: self.CopyBinaryFiles() return - if not self.IsLibrary and CreateLibraryCodeFile: - for LibraryAutoGen in self.LibraryAutoGenList: - LibraryAutoGen.CreateCodeFile() # Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping if not GlobalData.gUseHashCache and self.CanSkip(): return diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py index 6360d4cbd86b..9885c6a3a3bf 100644 --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py @@ -1081,14 +1081,14 @@ class PlatformAutoGen(AutoGen): def GetAllModuleInfo(self,WithoutPcd=True): ModuleLibs = set() for m in self.Platform.Modules: module_obj = self.BuildDatabase[m,self.Arch,self.BuildTarget,self.ToolChain] Libs = GetModuleLibInstances(module_obj, self.Platform, self.BuildDatabase, self.Arch,self.BuildTarget,self.ToolChain) - ModuleLibs.update( set([(l.MetaFile.File,l.MetaFile.Root,l.Arch,True) for l in Libs])) + ModuleLibs.update( set([(l.MetaFile.File,l.MetaFile.Root,l.MetaFile.Path,l.MetaFile.BaseName,l.MetaFile.OriginalPath,l.Arch,True) for l in Libs])) if WithoutPcd and module_obj.PcdIsDriver: continue - ModuleLibs.add((m.File,m.Root,module_obj.Arch,False)) + ModuleLibs.add((m.File,m.Root,m.Path,m.BaseName,m.OriginalPath,module_obj.Arch,bool(module_obj.LibraryClass))) return ModuleLibs ## Resolve the library classes in a module to library instances # diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index 3d083f4eaade..db210428e84c 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -28,10 +28,11 @@ from subprocess import Popen,PIPE from collections import OrderedDict, defaultdict from optparse import OptionParser from AutoGen.PlatformAutoGen import PlatformAutoGen from AutoGen.ModuleAutoGen import ModuleAutoGen from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen +from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager from AutoGen import GenMake from Common import Misc as Utils from Common.TargetTxtClassObject import TargetTxt from Common.ToolDefClassObject import ToolDef @@ -48,11 +49,11 @@ from BuildReport import BuildReport from GenPatchPcdTable.GenPatchPcdTable import PeImageClass,parsePcdInfoFromMapFile from PatchPcdValue.PatchPcdValue import PatchBinaryFile import Common.GlobalData as GlobalData from GenFds.GenFds import GenFds, GenFdsApi - +import multiprocessing as mp # Version and Copyright VersionNumber = "0.60" + ' ' + gBUILD_VERSION __version__ = "%prog Version " + VersionNumber __copyright__ = "Copyright (c) 2007 - 2018, Intel Corporation All rights reserved." @@ -341,13 +342,13 @@ class ModuleMakeUnit(BuildUnit): # # @param self The object pointer # @param Obj The ModuleAutoGen object the build is working on # @param Target The build target name, one of gSupportedTarget # - def __init__(self, Obj, Target): - Dependency = [ModuleMakeUnit(La, Target) for La in Obj.LibraryAutoGenList] - BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir) + def __init__(self, Obj, BuildCommand,Target): + Dependency = [ModuleMakeUnit(La, BuildCommand,Target) for La in Obj.LibraryAutoGenList] + BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir) if Target in [None, "", "all"]: self.Target = "tbuild" ## The smallest platform unit that can be built by nmake/make command in multi-thread build mode # @@ -362,14 +363,14 @@ class PlatformMakeUnit(BuildUnit): # # @param self The object pointer # @param Obj The PlatformAutoGen object the build is working on # @param Target The build target name, one of gSupportedTarget # - def __init__(self, Obj, Target): - Dependency = [ModuleMakeUnit(Lib, Target) for Lib in self.BuildObject.LibraryAutoGenList] - Dependency.extend([ModuleMakeUnit(Mod, Target) for Mod in self.BuildObject.ModuleAutoGenList]) - BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir) + def __init__(self, Obj, BuildCommand, Target): + Dependency = [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib in self.BuildObject.LibraryAutoGenList] + Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mod in self.BuildObject.ModuleAutoGenList]) + BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir) ## The class representing the task of a module build or platform build # # This class manages the build tasks in multi-thread build mode. Its jobs include # scheduling thread running, catching thread error, monitor the thread status, etc. @@ -822,12 +823,35 @@ class Build(): self.TargetTxt = TargetTxt self.ToolDef = ToolDef if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)): self.InitBuild() + self.AutoGenMgr = None EdkLogger.info("") os.chdir(self.WorkspaceDir) + def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList): + if SkipAutoGen: + return + feedback_q = mp.Queue() + file_lock = mp.Lock() + auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock) for _ in range(mp.cpu_count())] + self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q) + self.AutoGenMgr.start() + for w in auto_workers: + w.start() + if PcdMaList is not None: + for PcdMa in PcdMaList: + PcdMa.CreateCodeFile(True) + PcdMa.CreateMakeFile(GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[])) + PcdMa.CreateAsBuiltInf() + for w in auto_workers: + w.join() + rt = self.AutoGenMgr.Status + self.AutoGenMgr.kill() + self.AutoGenMgr.join() + self.AutoGenMgr = None + return rt ## Load configuration # # This method will parse target.txt and get the build configurations. # @@ -1188,30 +1212,29 @@ class Build(): # @param CreateDepModuleCodeFile Flag used to indicate creating code # for dependent modules/Libraries # @param CreateDepModuleMakeFile Flag used to indicate creating makefile # for dependent modules/Libraries # - def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand={}): + def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand=None, PcdMaList=None): if AutoGenObject is None: return False - + if FfsCommand is None: + FfsCommand = {} # skip file generation for cleanxxx targets, run and fds target if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile - if not self.SkipAutoGen or Target == 'genc': - self.Progress.Start("Generating code") - AutoGenObject.CreateCodeFile(CreateDepsCodeFile) - self.Progress.Stop("done!") - if Target == "genc": - return True + mqueue = mp.Queue() + for m in AutoGenObject.GetAllModuleInfo: + mqueue.put(m) - if not self.SkipAutoGen or Target == 'genmake': - self.Progress.Start("Generating makefile") - AutoGenObject.CreateMakeFile(CreateDepsMakeFile, FfsCommand) - self.Progress.Stop("done!") - if Target == "genmake": - return True + AutoGenObject.DataPipe.DataContainer = {"FfsCommand":FfsCommand} + self.Progress.Start("Generating makefile and code") + data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch)) + AutoGenObject.DataPipe.dump(data_pipe_file) + autogen_rt = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList) + self.Progress.Stop("done!") + return autogen_rt else: # always recreate top/platform makefile when clean, just in case of inconsistency AutoGenObject.CreateCodeFile(False) AutoGenObject.CreateMakeFile(False) @@ -1234,11 +1257,10 @@ class Build(): # build modules if BuildModule: BuildCommand = BuildCommand + [Target] LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir) - self.CreateAsBuiltInf() if GlobalData.gBinCacheDest: self.UpdateBuildCache() self.BuildModules = [] return True @@ -1255,11 +1277,10 @@ class Build(): NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild'] LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir) for Mod in AutoGenObject.ModuleBuildDirectoryList: NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, makefile)), 'pbuild'] LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir) - self.CreateAsBuiltInf() if GlobalData.gBinCacheDest: self.UpdateBuildCache() self.BuildModules = [] return True @@ -1713,11 +1734,11 @@ class Build(): continue if Ma.PcdIsDriver: Ma.PlatformInfo = Pa PcdMaList.append(Ma) self.BuildModules.append(Ma) - self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict) + self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList) # Create MAP file when Load Fix Address is enabled. if self.Target in ["", "all", "fds"]: for Arch in Wa.ArchList: GlobalData.gGlobalDefines['ARCH'] = Arch @@ -1848,11 +1869,11 @@ class Build(): GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL' self.AutoGenTime += int(round((time.time() - AutoGenStart))) MakeStart = time.time() for Ma in self.BuildModules: if not Ma.IsBinaryModule: - Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target)) + Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target)) # Break build if any build thread has error if BuildTask.HasError(): # we need a full version of makefile for platform ExitFlag.set() BuildTask.WaitForComplete() @@ -1978,18 +1999,19 @@ class Build(): self.LoadFixAddress = Wa.Platform.LoadFixAddress self.BuildReport.AddPlatformReport(Wa) Wa.CreateMakeFile(False) # Add ffs build to makefile - CmdListDict = None + CmdListDict = {} if GlobalData.gEnableGenfdsMultiThread and self.Fdf: CmdListDict = self._GenFfsCmd(Wa.ArchList) # multi-thread exit flag ExitFlag = threading.Event() ExitFlag.clear() self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) + BuildModules = [] for Arch in Wa.ArchList: PcdMaList = [] AutoGenStart = time.time() GlobalData.gGlobalDefines['ARCH'] = Arch Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) @@ -2022,38 +2044,35 @@ class Build(): else: if GlobalData.gBinCacheSource: EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds' - if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile - if not self.SkipAutoGen or self.Target == 'genc': - Ma.CreateCodeFile(True) - if self.Target == "genc": - continue - if not self.SkipAutoGen or self.Target == 'genmake': - if CmdListDict and self.Fdf and (Module.File, Arch) in CmdListDict: - Ma.CreateMakeFile(True, CmdListDict[Module.File, Arch]) - del CmdListDict[Module.File, Arch] - else: - Ma.CreateMakeFile(True) - if self.Target == "genmake": - continue - self.BuildModules.append(Ma) + BuildModules.append(Ma) # Initialize all modules in tracking to 'FAIL' if Ma.Arch not in GlobalData.gModuleBuildTracking: GlobalData.gModuleBuildTracking[Ma.Arch] = dict() if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]: GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL' + mqueue = mp.Queue() + for m in Pa.GetAllModuleInfo: + mqueue.put(m) + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) + Pa.DataPipe.dump(data_pipe_file) + autogen_rt = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList) self.Progress.Stop("done!") self.AutoGenTime += int(round((time.time() - AutoGenStart))) + if not autogen_rt: + return + for Arch in Wa.ArchList: MakeStart = time.time() - for Ma in self.BuildModules: + for Ma in BuildModules: # Generate build task for the module if not Ma.IsBinaryModule: - Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target)) + Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target)) # Break build if any build thread has error if BuildTask.HasError(): # we need a full version of makefile for platform ExitFlag.set() BuildTask.WaitForComplete() @@ -2078,11 +2097,10 @@ class Build(): # All modules have been put in build tasks queue. Tell task scheduler # to exit if all tasks are completed # ExitFlag.set() BuildTask.WaitForComplete() - self.CreateAsBuiltInf() if GlobalData.gBinCacheDest: self.UpdateBuildCache() self.BuildModules = [] self.MakeTime += int(round((time.time() - MakeContiue))) # @@ -2496,10 +2514,16 @@ def Main(): EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc()) else: EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False) ReturnCode = FORMAT_INVALID except KeyboardInterrupt: + if MyBuild is not None: + if MyBuild.AutoGenMgr: + MyBuild.AutoGenMgr.TerminateWorkers() + MyBuild.AutoGenMgr.kill() + # for multi-thread build exits safely + MyBuild.Relinquish() ReturnCode = ABORT_ERROR if Option is not None and Option.debug is not None: EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc()) except: if MyBuild is not None: -- 2.20.1.windows.1