From mboxrd@z Thu Jan 1 00:00:00 1970 Authentication-Results: mx.groups.io; dkim=missing; spf=fail (domain: intel.com, ip: , mailfrom: bob.c.feng@intel.com) Received: from mga01.intel.com (mga01.intel.com []) by groups.io with SMTP; Mon, 29 Jul 2019 01:45:14 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga004.jf.intel.com ([10.7.209.38]) by fmsmga101.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 29 Jul 2019 01:45:14 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.64,322,1559545200"; d="scan'208";a="322786672" Received: from shwdepsi1121.ccr.corp.intel.com ([10.239.158.47]) by orsmga004.jf.intel.com with ESMTP; 29 Jul 2019 01:45:13 -0700 From: "Bob Feng" To: devel@edk2.groups.io Cc: Liming Gao , Bob Feng Subject: [Patch 06/11] BaseTools: Add shared data for processes Date: Mon, 29 Jul 2019 16:44:51 +0800 Message-Id: <20190729084456.18844-7-bob.c.feng@intel.com> X-Mailer: git-send-email 2.20.1.windows.1 In-Reply-To: <20190729084456.18844-1-bob.c.feng@intel.com> References: <20190729084456.18844-1-bob.c.feng@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1875 Add shared data for autogen processes. Cc: Liming Gao Signed-off-by: Bob Feng --- BaseTools/Source/Python/AutoGen/AutoGenWorker.py | 3 ++- BaseTools/Source/Python/build/build.py | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py index edec346abd06..1419d92b5e39 100644 --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -52,18 +52,19 @@ class AutoGenManager(threading.Thread): for w in self.autogen_workers: if w.is_alive(): w.terminate() class AutoGenWorkerInProcess(mp.Process): - def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock): + def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock, share_data): mp.Process.__init__(self) self.module_queue = module_queue self.data_pipe_file_path =data_pipe_file_path self.data_pipe = None self.feedback_q = feedback_q self.PlatformMetaFileSet = {} self.file_lock = file_lock + self.share_data = share_data def GetPlatformMetaFile(self,filepath,root): try: return self.PlatformMetaFileSet[(filepath,root)] except: self.PlatformMetaFileSet[(filepath,root)] = filepath diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index 590584b3c67f..d49554ec0282 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -50,10 +50,11 @@ from GenPatchPcdTable.GenPatchPcdTable import PeImageClass,parsePcdInfoFromMapFi from PatchPcdValue.PatchPcdValue import PatchBinaryFile import Common.GlobalData as GlobalData from GenFds.GenFds import GenFds, GenFdsApi import multiprocessing as mp +from multiprocessing import Manager # Version and Copyright VersionNumber = "0.60" + ' ' + gBUILD_VERSION __version__ = "%prog Version " + VersionNumber __copyright__ = "Copyright (c) 2007 - 2018, Intel Corporation All rights reserved." @@ -826,16 +827,17 @@ class Build(): self.InitBuild() self.AutoGenMgr = None EdkLogger.info("") os.chdir(self.WorkspaceDir) - def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList): + self.share_data = Manager().dict() + def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,share_data): if SkipAutoGen: return feedback_q = mp.Queue() file_lock = mp.Lock() - auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock) for _ in range(self.ThreadNumber)] + auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock,share_data) for _ in range(self.ThreadNumber)] self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q) self.AutoGenMgr.start() for w in auto_workers: w.start() if PcdMaList is not None: @@ -1228,11 +1230,11 @@ class Build(): AutoGenObject.DataPipe.DataContainer = {"FfsCommand":FfsCommand} self.Progress.Start("Generating makefile and code") data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch)) AutoGenObject.DataPipe.dump(data_pipe_file) - autogen_rt = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList) + autogen_rt = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data) self.Progress.Stop("done!") return autogen_rt else: # always recreate top/platform makefile when clean, just in case of inconsistency AutoGenObject.CreateCodeFile(False) @@ -2060,11 +2062,11 @@ class Build(): mqueue = mp.Queue() for m in Pa.GetAllModuleInfo: mqueue.put(m) data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) Pa.DataPipe.dump(data_pipe_file) - autogen_rt = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList) + autogen_rt = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data) self.Progress.Stop("done!") self.AutoGenTime += int(round((time.time() - AutoGenStart))) if not autogen_rt: return for Arch in Wa.ArchList: -- 2.20.1.windows.1