public inbox for devel@edk2.groups.io
 help / color / mirror / Atom feed
* [Patch] BaseTools: Improve the method of checking queue empty
@ 2020-08-03  2:03 Bob Feng
  2020-08-04  9:00 ` [edk2-devel] " Liming Gao
  0 siblings, 1 reply; 2+ messages in thread
From: Bob Feng @ 2020-08-03  2:03 UTC (permalink / raw)
  To: devel; +Cc: Feng, Bob C, Liming Gao, Yuwei Chen, Lucy Yan

From: "Feng, Bob C" <bob.c.feng@intel.com>

Ref: https://bugzilla.tianocore.org/show_bug.cgi?id=2807

The Queue.empty() method is not reliable in the multiple
process runtime environment. This patch uses a new method
to check if all modules are processed and workers need
to be stopped. That is to add a None item at the bottom
of the queue. Worker check if it gets that None item to
know if all the module is processed.

Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Cc: Yuwei Chen <yuwei.chen@intel.com>
Cc: Lucy Yan <lucyyan@google.com>
---
 .../Source/Python/AutoGen/AutoGenWorker.py    | 26 ++++++++++++++-----
 BaseTools/Source/Python/build/build.py        |  3 ++-
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
index 563d91b421..017f676399 100755
--- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
@@ -22,10 +22,11 @@ except:
     from Queue import Empty
 import traceback
 import sys
 from AutoGen.DataPipe import MemoryDataPipe
 import logging
+import time
 
 def clearQ(q):
     try:
         while True:
             q.get_nowait()
@@ -109,11 +110,15 @@ class AutoGenManager(threading.Thread):
                 badnews = self.feedback_q.get()
                 if badnews is None:
                     break
                 if badnews == "Done":
                     fin_num += 1
+                elif badnews == "QueueEmpty":
+                    EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews))
+                    self.TerminateWorkers()
                 else:
+                    EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews))
                     self.Status = False
                     self.TerminateWorkers()
                 if fin_num == len(self.autogen_workers):
                     self.clearQueue()
                     for w in self.autogen_workers:
@@ -225,16 +230,25 @@ class AutoGenWorkerInProcess(mp.Process):
                 FfsCmd = {}
             GlobalData.FfsCmd = FfsCmd
             PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),
                                              self.data_pipe.Get("P_Info").get("WorkspaceDir"))
             while True:
-                if self.module_queue.empty():
-                    break
                 if self.error_event.is_set():
                     break
                 module_count += 1
-                module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()
+                try:
+                    module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()
+                except Empty:
+                    EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Fake Empty."))
+                    time.sleep(0.01)
+                    continue
+                if module_file is None:
+                    EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Worker get the last item in the queue."))
+                    self.feedback_q.put("QueueEmpty")
+                    time.sleep(0.01)
+                    continue
+
                 modulefullpath = os.path.join(module_root,module_file)
                 taskname = " : ".join((modulefullpath,module_arch))
                 module_metafile = PathClass(module_file,module_root)
                 if module_path:
                     module_metafile.Path = module_path
@@ -278,15 +292,15 @@ class AutoGenWorkerInProcess(mp.Process):
                         self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True))
                         continue
                     else:
                         self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False))
 
-        except Empty:
-            pass
-        except:
+        except Exception as e:
+            EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), str(e)))
             self.feedback_q.put(taskname)
         finally:
+            EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Done"))
             self.feedback_q.put("Done")
             self.cache_q.put("CacheDone")
 
     def printStatus(self):
         print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 1ab1e60a64..59ceacfed0 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -1215,11 +1215,11 @@ class Build():
         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
             # for target which must generate AutoGen code and makefile
             mqueue = mp.Queue()
             for m in AutoGenObject.GetAllModuleInfo:
                 mqueue.put(m)
-
+            mqueue.put((None,None,None,None,None,None,None))
             AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}
             AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}
             AutoGenObject.CreateLibModuelDirs()
             AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
             AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
@@ -2172,10 +2172,11 @@ class Build():
                                   ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
                 self.AllModules.add(Ma)
             data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
             Pa.DataPipe.dump(data_pipe_file)
 
+            mqueue.put((None,None,None,None,None,None,None))
             autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)
 
             if not autogen_rt:
                 self.AutoGenMgr.TerminateWorkers()
                 self.AutoGenMgr.join(1)
-- 
2.20.1.windows.1


^ permalink raw reply related	[flat|nested] 2+ messages in thread
end of thread, other threads:[~2020-08-04  9:00 UTC | newest]

Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2020-08-03  2:03 [Patch] BaseTools: Improve the method of checking queue empty Bob Feng
2020-08-04  9:00 ` [edk2-devel] " Liming Gao

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox