[Patch] BaseTools: Support multi thread build Basetool on Windows

* [Patch] BaseTools: Support multi thread build Basetool on Windows
@ 2018-08-28 15:26 Liming Gao
  2018-08-28 16:09 ` Carsey, Jaben
  0 siblings, 1 reply; 3+ messages in thread
From: Liming Gao @ 2018-08-28 15:26 UTC (permalink / raw)
  To: edk2-devel; +Cc: Dongao Guo

From: Dongao Guo <dongao.guo@intel.com>

Add NmakeSubdirs.py to replace NmakeSubdirs.bat in VS Makefile. This script will
invoke nmake in multi thread mode. It can save more than half time of BaseTools
C clean build.
GCC make supports multiple thread in make phase. So, GNNmakefile doesn't need apply
this script.

single task or job=1:
    just single thread and invoke subprocess,subprocess will use
    system.stdout to print output.
multi task:
    thread number is logic cpu count.All subprocess output will pass to
    python script by PIPE and then script print it to system.stdout.

Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Dongao Guo<dongao.guo@intel.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>
Test-by: Liming Gao <liming.gao@intel.com>
---
 BaseTools/Makefile                           |  12 +-
 BaseTools/Source/C/Makefile                  |  14 +--
 BaseTools/Source/C/Makefiles/NmakeSubdirs.py | 169 +++++++++++++++++++++++++++
 3 files changed, 182 insertions(+), 13 deletions(-)
 create mode 100644 BaseTools/Source/C/Makefiles/NmakeSubdirs.py

diff --git a/BaseTools/Makefile b/BaseTools/Makefile
index 3736d85..b98cd85 100644
--- a/BaseTools/Makefile
+++ b/BaseTools/Makefile
@@ -1,7 +1,7 @@
 ## @file
 # Windows makefile for Base Tools project build.
 #
-# Copyright (c) 2007 - 2017, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
 # This program and the accompanying materials
 # are licensed and made available under the terms and conditions of the BSD License
 # which accompanies this distribution.  The full text of the license may be found at
@@ -20,19 +20,19 @@ SUBDIRS = $(BASE_TOOLS_PATH)\Source\C $(BASE_TOOLS_PATH)\Source\Python
 all: c python
 
 c :
-  @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat all $(BASE_TOOLS_PATH)\Source\C
+  @$(PYTHON_HOME)\python.exe $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py  all $(BASE_TOOLS_PATH)\Source\C
 
 python:
-  @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat all $(BASE_TOOLS_PATH)\Source\Python
+  @$(PYTHON_HOME)\python.exe $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py  all $(BASE_TOOLS_PATH)\Source\Python
 
 subdirs: $(SUBDIRS)
-  @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat all $**
+  @$(PYTHON_HOME)\python.exe $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py  all $**
 
 .PHONY: clean
 clean:
-  @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat clean $(SUBDIRS)
+  $(PYTHON_HOME)\python.exe $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py clean $(SUBDIRS)
 
 .PHONY: cleanall
 cleanall:
-  @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat cleanall $(SUBDIRS)
+  $(PYTHON_HOME)\python.exe $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py  cleanall $(SUBDIRS)
 
diff --git a/BaseTools/Source/C/Makefile b/BaseTools/Source/C/Makefile
index 5428180..1246d23 100644
--- a/BaseTools/Source/C/Makefile
+++ b/BaseTools/Source/C/Makefile
@@ -1,7 +1,7 @@
 ## @file
 # Windows makefile for C tools build.
 #
-# Copyright (c) 2009 - 2017, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2009 - 2018, Intel Corporation. All rights reserved.<BR>
 # This program and the accompanying materials
 # are licensed and made available under the terms and conditions of the BSD License
 # which accompanies this distribution.  The full text of the license may be found at
@@ -16,7 +16,7 @@ HOST_ARCH = IA32
 
 LIBRARIES = Common
 APPLICATIONS = \
-  BootSectImage \
+  VfrCompile \
   BrotliCompress \
   EfiLdrImage \
   EfiRom \
@@ -32,7 +32,7 @@ APPLICATIONS = \
   Split \
   TianoCompress \
   VolInfo \
-  VfrCompile \
+  BootSectImage \
   DevicePath
 
 all: libs apps install
@@ -43,7 +43,7 @@ libs: $(LIBRARIES)
 	@echo # Build libraries
 	@echo ######################
 	@if not exist $(LIB_PATH) mkdir $(LIB_PATH)
-	@Makefiles\NmakeSubdirs.bat all $**
+	@$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py all $**
 
 apps: $(APPLICATIONS)
 	@echo.
@@ -51,7 +51,7 @@ apps: $(APPLICATIONS)
 	@echo # Build executables
 	@echo ######################
 	@if not exist $(BIN_PATH) mkdir $(BIN_PATH)
-	@Makefiles\NmakeSubdirs.bat all $**
+	@$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py all $**
 
 install: $(LIB_PATH) $(BIN_PATH)
 	@echo.
@@ -65,11 +65,11 @@ install: $(LIB_PATH) $(BIN_PATH)
 
 .PHONY: clean
 clean:
-  @Makefiles\NmakeSubdirs.bat clean $(LIBRARIES) $(APPLICATIONS)
+  @$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py clean $(LIBRARIES) $(APPLICATIONS)
 
 .PHONY: cleanall
 cleanall:
-  @Makefiles\NmakeSubdirs.bat cleanall $(LIBRARIES) $(APPLICATIONS)
+  @$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py cleanall $(LIBRARIES) $(APPLICATIONS)
 
 !INCLUDE Makefiles\ms.rule
 
diff --git a/BaseTools/Source/C/Makefiles/NmakeSubdirs.py b/BaseTools/Source/C/Makefiles/NmakeSubdirs.py
new file mode 100644
index 0000000..29bb5df
--- /dev/null
+++ b/BaseTools/Source/C/Makefiles/NmakeSubdirs.py
@@ -0,0 +1,169 @@
+# @file NmakeSubdirs.py
+# This script support parallel build for nmake in windows environment.
+# It supports Python2.x and Python3.x both.
+#
+#  Copyright (c) 2018, Intel Corporation. All rights reserved.<BR>
+#
+#  This program and the accompanying materials
+#  are licensed and made available under the terms and conditions of the BSD License
+#  which accompanies this distribution.  The full text of the license may be found at
+#  http://opensource.org/licenses/bsd-license.php
+#
+#  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+#
+
+#
+# Import Modules
+#
+
+from __future__ import print_function
+import argparse
+import threading
+import time
+import os
+import subprocess
+import multiprocessing
+import copy
+import sys
+__prog__        = 'NmakeSubdirs'
+__version__     = '%s Version %s' % (__prog__, '0.10 ')
+__copyright__   = 'Copyright (c) 2018, Intel Corporation. All rights reserved.'
+__description__ = 'Replace for NmakeSubdirs.bat in windows ,support parallel build for nmake.\n'
+
+cpu_count = multiprocessing.cpu_count()
+output_lock = threading.Lock()
+def RunCommand(WorkDir=None, *Args, **kwargs):
+    if WorkDir is None:
+        WorkDir = os.curdir
+    if "stderr" not in kwargs:
+        kwargs["stderr"] = subprocess.STDOUT
+    if "stdout" not in kwargs:
+        kwargs["stdout"] = subprocess.PIPE
+    p = subprocess.Popen(Args, cwd=WorkDir, stderr=kwargs["stderr"], stdout=kwargs["stdout"])
+    stdout, stderr = p.communicate()
+    message = ""
+    if stdout is not None:
+        message = stdout.decode() #for compatibility in python 2 and 3
+
+    if p.returncode != 0:
+        raise RuntimeError("Error while execute command \'{0}\' in direcotry {1}\n{2}".format(" ".join(Args), WorkDir, message))
+
+    output_lock.acquire(True)
+    print("execute command \"{0}\" in directory {1}".format(" ".join(Args), WorkDir))
+    print(message)
+    output_lock.release()
+
+    return p.returncode, stdout
+
+class TaskUnit(object):
+    def __init__(self, func, args, kwargs):
+        self.func = func
+        self.args = args
+        self.kwargs = kwargs
+
+    def __eq__(self, other):
+        return id(self).__eq__(id(other))
+
+    def run(self):
+        return self.func(*self.args, **self.kwargs)
+
+    def __str__(self):
+        para = list(self.args)
+        para.extend("{0}={1}".format(k, v)for k, v in self.kwargs.items())
+
+        return "{0}({1})".format(self.func.__name__, ",".join(para))
+
+class ThreadControl(object):
+
+    def __init__(self, maxthread):
+        self._processNum = maxthread
+        self.pending = []
+        self.running = []
+        self.pendingLock = threading.Lock()
+        self.runningLock = threading.Lock()
+        self.error = False
+        self.errorLock = threading.Lock()
+        self.errorMsg = "errorMsg"
+
+    def addTask(self, func, *args, **kwargs):
+        self.pending.append(TaskUnit(func, args, kwargs))
+
+    def waitComplete(self):
+        self._schedule.join()
+
+    def startSchedule(self):
+        self._schedule = threading.Thread(target=self.Schedule)
+        self._schedule.start()
+
+    def Schedule(self):
+        for i in range(self._processNum):
+            task = threading.Thread(target=self.startTask)
+            task.daemon = False
+            self.running.append(task)
+
+        self.runningLock.acquire(True)
+        for thread in self.running:
+            thread.start()
+        self.runningLock.release()
+
+        while len(self.running) > 0:
+            time.sleep(0.1)
+        if self.error:
+            print("subprocess not exit sucessfully")
+            print(self.errorMsg)
+
+    def startTask(self):
+        while True:
+            if self.error:
+                break
+            self.pendingLock.acquire(True)
+            if len(self.pending) == 0:
+                self.pendingLock.release()
+                break
+            task = self.pending.pop(0)
+            self.pendingLock.release()
+            try:
+                task.run()
+            except RuntimeError as e:
+                if self.error: break
+                self.errorLock.acquire(True)
+                self.error = True
+                self.errorMsg = str(e)
+                time.sleep(0.1)
+                self.errorLock.release()
+                break
+
+        self.runningLock.acquire(True)
+        self.running.remove(threading.currentThread())
+        self.runningLock.release()
+
+def Run():
+    curdir = os.path.abspath(os.curdir)
+    if len(args.subdirs) == 1:
+        args.jobs = 1
+    if args.jobs == 1:
+        try:
+            for dir in args.subdirs:
+                RunCommand(os.path.join(curdir, dir), "nmake", args.target, stdout=sys.stdout, stderr=subprocess.STDOUT)
+        except RuntimeError:
+            exit(1)
+    else:
+        controller = ThreadControl(args.jobs)
+        for dir in args.subdirs:
+            controller.addTask(RunCommand, os.path.join(curdir, dir), "nmake", args.target)
+        controller.startSchedule()
+        controller.waitComplete()
+        if controller.error:
+            exit(1)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(prog=__prog__, description=__description__ + __copyright__, conflict_handler='resolve')
+
+    parser.add_argument("target", help="the target for nmake")
+    parser.add_argument("subdirs", nargs="+", help="the relative dir path of makefile")
+    parser.add_argument("--jobs", type=int, dest="jobs", default=cpu_count, help="thread number")
+    parser.add_argument('--version', action='version', version=__version__)
+    args = parser.parse_args()
+    Run()
+
-- 
2.10.0.windows.1



^ permalink raw reply related	[flat|nested] 3+ messages in thread