From mboxrd@z Thu Jan 1 00:00:00 1970 Authentication-Results: mx.groups.io; dkim=missing; spf=fail (domain: intel.com, ip: , mailfrom: bob.c.feng@intel.com) Received: from mga02.intel.com (mga02.intel.com []) by groups.io with SMTP; Wed, 31 Jul 2019 20:03:54 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga004.jf.intel.com ([10.7.209.38]) by orsmga101.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 31 Jul 2019 20:03:54 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.64,332,1559545200"; d="scan'208";a="324044556" Received: from shwdepsi1121.ccr.corp.intel.com ([10.239.158.47]) by orsmga004.jf.intel.com with ESMTP; 31 Jul 2019 20:03:53 -0700 From: "Bob Feng" To: devel@edk2.groups.io Cc: Liming Gao , Bob Feng Subject: [Patch 10/10 V6] BaseTools: Enable block queue log agent. Date: Thu, 1 Aug 2019 11:03:33 +0800 Message-Id: <20190801030333.8588-11-bob.c.feng@intel.com> X-Mailer: git-send-email 2.20.1.windows.1 In-Reply-To: <20190801030333.8588-1-bob.c.feng@intel.com> References: <20190801030333.8588-1-bob.c.feng@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1875 To support Ctrl+S and Ctrl+Q, we enable block queue for log. Cc: Liming Gao Signed-off-by: Bob Feng --- .../Source/Python/AutoGen/AutoGenWorker.py | 6 ++- BaseTools/Source/Python/Common/EdkLogger.py | 10 +++-- BaseTools/Source/Python/build/build.py | 39 ++++++++++--------- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py index 1f388ee178be..635ddfe5203f 100644 --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -21,17 +21,18 @@ try: except: from Queue import Empty import traceback import sys from AutoGen.DataPipe import MemoryDataPipe +import logging + def clearQ(q): try: while True: q.get_nowait() except Empty: pass -import logging class LogAgent(threading.Thread): def __init__(self,log_q,log_level,log_file=None): super(LogAgent,self).__init__() self.log_q = log_q @@ -121,13 +122,14 @@ class AutoGenManager(threading.Thread): except Exception: return def clearQueue(self): taskq = self.autogen_workers[0].module_queue + logq = self.autogen_workers[0].log_q clearQ(taskq) clearQ(self.feedback_q) - + clearQ(logq) def TerminateWorkers(self): self.error_event.set() def kill(self): self.feedback_q.put(None) class AutoGenWorkerInProcess(mp.Process): diff --git a/BaseTools/Source/Python/Common/EdkLogger.py b/BaseTools/Source/Python/Common/EdkLogger.py index 15fd1458a95a..06da4a9d0a1d 100644 --- a/BaseTools/Source/Python/Common/EdkLogger.py +++ b/BaseTools/Source/Python/Common/EdkLogger.py @@ -93,11 +93,13 @@ except: """ try: self.enqueue(self.prepare(record)) except Exception: self.handleError(record) - +class BlockQueueHandler(QueueHandler): + def enqueue(self, record): + self.queue.put(record,True) ## Log level constants DEBUG_0 = 1 DEBUG_1 = 2 DEBUG_2 = 3 DEBUG_3 = 4 @@ -290,23 +292,23 @@ def LogClientInitialize(log_q): # Since we use different format to log different levels of message into different # place (stdout or stderr), we have to use different "Logger" objects to do this. # # For DEBUG level (All DEBUG_0~9 are applicable) _DebugLogger.setLevel(INFO) - _DebugChannel = QueueHandler(log_q) + _DebugChannel = BlockQueueHandler(log_q) _DebugChannel.setFormatter(_DebugFormatter) _DebugLogger.addHandler(_DebugChannel) # For VERBOSE, INFO, WARN level _InfoLogger.setLevel(INFO) - _InfoChannel = QueueHandler(log_q) + _InfoChannel = BlockQueueHandler(log_q) _InfoChannel.setFormatter(_InfoFormatter) _InfoLogger.addHandler(_InfoChannel) # For ERROR level _ErrorLogger.setLevel(INFO) - _ErrorCh = QueueHandler(log_q) + _ErrorCh = BlockQueueHandler(log_q) _ErrorCh.setFormatter(_ErrorFormatter) _ErrorLogger.addHandler(_ErrorCh) ## Set log level # diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index 1c03f063668b..f06fd47a8916 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -707,11 +707,11 @@ class Build(): self.Fdf = BuildOptions.FdfFile self.FdList = BuildOptions.RomImage self.FvList = BuildOptions.FvImage self.CapList = BuildOptions.CapName self.SilentMode = BuildOptions.SilentMode - self.ThreadNumber = BuildOptions.ThreadNumber + self.ThreadNumber = 1 self.SkipAutoGen = BuildOptions.SkipAutoGen self.Reparse = BuildOptions.Reparse self.SkuId = BuildOptions.SkuId if self.SkuId: GlobalData.gSKUID_CMD = self.SkuId @@ -881,23 +881,10 @@ class Build(): ToolChainFamily.append(TAB_COMPILER_MSFT) else: ToolChainFamily.append(ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool]) self.ToolChainFamily = ToolChainFamily - if self.ThreadNumber is None: - self.ThreadNumber = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER] - if self.ThreadNumber == '': - self.ThreadNumber = 0 - else: - self.ThreadNumber = int(self.ThreadNumber, 0) - - if self.ThreadNumber == 0: - try: - self.ThreadNumber = multiprocessing.cpu_count() - except (ImportError, NotImplementedError): - self.ThreadNumber = 1 - if not self.PlatformFile: PlatformFile = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_ACTIVE_PLATFORM] if not PlatformFile: # Try to find one in current directory WorkingDirectory = os.getcwd() @@ -911,11 +898,11 @@ class Build(): else: EdkLogger.error("build", RESOURCE_NOT_AVAILABLE, ExtraData="No active platform specified in target.txt or command line! Nothing can be built.\n") self.PlatformFile = PathClass(NormFile(PlatformFile, self.WorkspaceDir), self.WorkspaceDir) - + self.ThreadNumber = ThreadNum() ## Initialize build configuration # # This method will parse DSC file and merge the configurations from # command line and target.txt, then get the final build configurations. # @@ -2052,16 +2039,17 @@ class Build(): for m in Pa.GetAllModuleInfo: mqueue.put(m) data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) Pa.DataPipe.dump(data_pipe_file) autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data) - self.Progress.Stop("done!") - self.AutoGenTime += int(round((time.time() - AutoGenStart))) + if not autogen_rt: self.AutoGenMgr.TerminateWorkers() self.AutoGenMgr.join(0.1) raise FatalError(errorcode) + self.AutoGenTime += int(round((time.time() - AutoGenStart))) + self.Progress.Stop("done!") for Arch in Wa.ArchList: MakeStart = time.time() for Ma in BuildModules: # Generate build task for the module if not Ma.IsBinaryModule: @@ -2292,27 +2280,42 @@ def LogBuildTime(Time): else: TimeDurStr = time.strftime("%H:%M:%S", TimeDur) return TimeDurStr else: return None +def ThreadNum(): + ThreadNumber = BuildOption.ThreadNumber + if ThreadNumber is None: + ThreadNumber = TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER] + if ThreadNumber == '': + ThreadNumber = 0 + else: + ThreadNumber = int(ThreadNumber, 0) + if ThreadNumber == 0: + try: + ThreadNumber = multiprocessing.cpu_count() + except (ImportError, NotImplementedError): + ThreadNumber = 1 + return ThreadNumber ## Tool entrance method # # This method mainly dispatch specific methods per the command line options. # If no error found, return zero value so the caller of this tool can know # if it's executed successfully or not. # # @retval 0 Tool was successful # @retval 1 Tool failed # +LogQMaxSize = ThreadNum() * 10 def Main(): StartTime = time.time() # # Create a log Queue # - LogQ = mp.Queue() + LogQ = mp.Queue(LogQMaxSize) # Initialize log system EdkLogger.LogClientInitialize(LogQ) GlobalData.gCommand = sys.argv[1:] # # Parse the options and args -- 2.20.1.windows.1