From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mga06.intel.com (mga06.intel.com [134.134.136.31]) by mx.groups.io with SMTP id smtpd.web11.2816.1596420223572388601 for ; Sun, 02 Aug 2020 19:03:43 -0700 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 134.134.136.31, mailfrom: bob.c.feng@intel.com) IronPort-SDR: 76GwbL73+51deQDD3F40f0Ng46GAn4lJwJh9r+/FyKs5j7ECjPfcitbVDkDaYP7n0eqOaczaBS HUGMMHiZvAcg== X-IronPort-AV: E=McAfee;i="6000,8403,9701"; a="213567613" X-IronPort-AV: E=Sophos;i="5.75,428,1589266800"; d="scan'208";a="213567613" X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga006.fm.intel.com ([10.253.24.20]) by orsmga104.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 02 Aug 2020 19:03:42 -0700 IronPort-SDR: cuTLmzBRY0Yfw1dR7RYyIRc8GfiJ+I9/ju+bmOykAP5lJw70edDbntomYsD1rUsMqeFSNJmO5x 4Gu80uo9gPhA== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.75,428,1589266800"; d="scan'208";a="491657468" Received: from shwdepsi1121.ccr.corp.intel.com ([10.239.158.83]) by fmsmga006.fm.intel.com with ESMTP; 02 Aug 2020 19:03:40 -0700 From: "Bob Feng" To: devel@edk2.groups.io Cc: "Feng, Bob C" , Liming Gao , Yuwei Chen , Lucy Yan Subject: [Patch] BaseTools: Improve the method of checking queue empty Date: Mon, 3 Aug 2020 10:03:38 +0800 Message-Id: <20200803020338.24712-1-bob.c.feng@intel.com> X-Mailer: git-send-email 2.20.1.windows.1 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit From: "Feng, Bob C" Ref: https://bugzilla.tianocore.org/show_bug.cgi?id=2807 The Queue.empty() method is not reliable in the multiple process runtime environment. This patch uses a new method to check if all modules are processed and workers need to be stopped. That is to add a None item at the bottom of the queue. Worker check if it gets that None item to know if all the module is processed. Signed-off-by: Bob Feng Cc: Liming Gao Cc: Yuwei Chen Cc: Lucy Yan --- .../Source/Python/AutoGen/AutoGenWorker.py | 26 ++++++++++++++----- BaseTools/Source/Python/build/build.py | 3 ++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py index 563d91b421..017f676399 100755 --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -22,10 +22,11 @@ except: from Queue import Empty import traceback import sys from AutoGen.DataPipe import MemoryDataPipe import logging +import time def clearQ(q): try: while True: q.get_nowait() @@ -109,11 +110,15 @@ class AutoGenManager(threading.Thread): badnews = self.feedback_q.get() if badnews is None: break if badnews == "Done": fin_num += 1 + elif badnews == "QueueEmpty": + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews)) + self.TerminateWorkers() else: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews)) self.Status = False self.TerminateWorkers() if fin_num == len(self.autogen_workers): self.clearQueue() for w in self.autogen_workers: @@ -225,16 +230,25 @@ class AutoGenWorkerInProcess(mp.Process): FfsCmd = {} GlobalData.FfsCmd = FfsCmd PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"), self.data_pipe.Get("P_Info").get("WorkspaceDir")) while True: - if self.module_queue.empty(): - break if self.error_event.is_set(): break module_count += 1 - module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait() + try: + module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait() + except Empty: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Fake Empty.")) + time.sleep(0.01) + continue + if module_file is None: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Worker get the last item in the queue.")) + self.feedback_q.put("QueueEmpty") + time.sleep(0.01) + continue + modulefullpath = os.path.join(module_root,module_file) taskname = " : ".join((modulefullpath,module_arch)) module_metafile = PathClass(module_file,module_root) if module_path: module_metafile.Path = module_path @@ -278,15 +292,15 @@ class AutoGenWorkerInProcess(mp.Process): self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True)) continue else: self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False)) - except Empty: - pass - except: + except Exception as e: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), str(e))) self.feedback_q.put(taskname) finally: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Done")) self.feedback_q.put("Done") self.cache_q.put("CacheDone") def printStatus(self): print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache()))) diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index 1ab1e60a64..59ceacfed0 100755 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -1215,11 +1215,11 @@ class Build(): if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile mqueue = mp.Queue() for m in AutoGenObject.GetAllModuleInfo: mqueue.put(m) - + mqueue.put((None,None,None,None,None,None,None)) AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target} AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp} AutoGenObject.CreateLibModuelDirs() AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList} AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList} @@ -2172,10 +2172,11 @@ class Build(): ToolChain, Arch, self.PlatformFile,Pa.DataPipe) self.AllModules.add(Ma) data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) Pa.DataPipe.dump(data_pipe_file) + mqueue.put((None,None,None,None,None,None,None)) autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, cqueue) if not autogen_rt: self.AutoGenMgr.TerminateWorkers() self.AutoGenMgr.join(1) -- 2.20.1.windows.1