From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=134.134.136.24; helo=mga09.intel.com; envelope-from=liming.gao@intel.com; receiver=edk2-devel@lists.01.org Received: from mga09.intel.com (mga09.intel.com [134.134.136.24]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ml01.01.org (Postfix) with ESMTPS id E4A60210F75D7 for ; Tue, 28 Aug 2018 08:28:07 -0700 (PDT) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by orsmga102.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 28 Aug 2018 08:28:07 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.53,299,1531810800"; d="scan'208";a="257916182" Received: from lgao4-mobl1.ccr.corp.intel.com ([10.249.173.21]) by fmsmga005.fm.intel.com with ESMTP; 28 Aug 2018 08:28:06 -0700 From: Liming Gao To: edk2-devel@lists.01.org Cc: Dongao Guo Date: Tue, 28 Aug 2018 23:26:56 +0800 Message-Id: <20180828152656.17596-1-liming.gao@intel.com> X-Mailer: git-send-email 2.16.2.windows.1 Subject: [Patch] BaseTools: Support multi thread build Basetool on Windows X-BeenThere: edk2-devel@lists.01.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: EDK II Development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 28 Aug 2018 15:28:08 -0000 From: Dongao Guo Add NmakeSubdirs.py to replace NmakeSubdirs.bat in VS Makefile. This script will invoke nmake in multi thread mode. It can save more than half time of BaseTools C clean build. GCC make supports multiple thread in make phase. So, GNNmakefile doesn't need apply this script. single task or job=1: just single thread and invoke subprocess,subprocess will use system.stdout to print output. multi task: thread number is logic cpu count.All subprocess output will pass to python script by PIPE and then script print it to system.stdout. Contributed-under: TianoCore Contribution Agreement 1.1 Signed-off-by: Dongao Guo Reviewed-by: Liming Gao Test-by: Liming Gao --- BaseTools/Makefile | 12 +- BaseTools/Source/C/Makefile | 14 +-- BaseTools/Source/C/Makefiles/NmakeSubdirs.py | 169 +++++++++++++++++++++++++++ 3 files changed, 182 insertions(+), 13 deletions(-) create mode 100644 BaseTools/Source/C/Makefiles/NmakeSubdirs.py diff --git a/BaseTools/Makefile b/BaseTools/Makefile index 3736d85..b98cd85 100644 --- a/BaseTools/Makefile +++ b/BaseTools/Makefile @@ -1,7 +1,7 @@ ## @file # Windows makefile for Base Tools project build. # -# Copyright (c) 2007 - 2017, Intel Corporation. All rights reserved.
+# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.
# This program and the accompanying materials # are licensed and made available under the terms and conditions of the BSD License # which accompanies this distribution. The full text of the license may be found at @@ -20,19 +20,19 @@ SUBDIRS = $(BASE_TOOLS_PATH)\Source\C $(BASE_TOOLS_PATH)\Source\Python all: c python c : - @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat all $(BASE_TOOLS_PATH)\Source\C + @$(PYTHON_HOME)\python.exe $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py all $(BASE_TOOLS_PATH)\Source\C python: - @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat all $(BASE_TOOLS_PATH)\Source\Python + @$(PYTHON_HOME)\python.exe $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py all $(BASE_TOOLS_PATH)\Source\Python subdirs: $(SUBDIRS) - @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat all $** + @$(PYTHON_HOME)\python.exe $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py all $** .PHONY: clean clean: - @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat clean $(SUBDIRS) + $(PYTHON_HOME)\python.exe $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py clean $(SUBDIRS) .PHONY: cleanall cleanall: - @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat cleanall $(SUBDIRS) + $(PYTHON_HOME)\python.exe $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py cleanall $(SUBDIRS) diff --git a/BaseTools/Source/C/Makefile b/BaseTools/Source/C/Makefile index 5428180..1246d23 100644 --- a/BaseTools/Source/C/Makefile +++ b/BaseTools/Source/C/Makefile @@ -1,7 +1,7 @@ ## @file # Windows makefile for C tools build. # -# Copyright (c) 2009 - 2017, Intel Corporation. All rights reserved.
+# Copyright (c) 2009 - 2018, Intel Corporation. All rights reserved.
# This program and the accompanying materials # are licensed and made available under the terms and conditions of the BSD License # which accompanies this distribution. The full text of the license may be found at @@ -16,7 +16,7 @@ HOST_ARCH = IA32 LIBRARIES = Common APPLICATIONS = \ - BootSectImage \ + VfrCompile \ BrotliCompress \ EfiLdrImage \ EfiRom \ @@ -32,7 +32,7 @@ APPLICATIONS = \ Split \ TianoCompress \ VolInfo \ - VfrCompile \ + BootSectImage \ DevicePath all: libs apps install @@ -43,7 +43,7 @@ libs: $(LIBRARIES) @echo # Build libraries @echo ###################### @if not exist $(LIB_PATH) mkdir $(LIB_PATH) - @Makefiles\NmakeSubdirs.bat all $** + @$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py all $** apps: $(APPLICATIONS) @echo. @@ -51,7 +51,7 @@ apps: $(APPLICATIONS) @echo # Build executables @echo ###################### @if not exist $(BIN_PATH) mkdir $(BIN_PATH) - @Makefiles\NmakeSubdirs.bat all $** + @$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py all $** install: $(LIB_PATH) $(BIN_PATH) @echo. @@ -65,11 +65,11 @@ install: $(LIB_PATH) $(BIN_PATH) .PHONY: clean clean: - @Makefiles\NmakeSubdirs.bat clean $(LIBRARIES) $(APPLICATIONS) + @$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py clean $(LIBRARIES) $(APPLICATIONS) .PHONY: cleanall cleanall: - @Makefiles\NmakeSubdirs.bat cleanall $(LIBRARIES) $(APPLICATIONS) + @$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py cleanall $(LIBRARIES) $(APPLICATIONS) !INCLUDE Makefiles\ms.rule diff --git a/BaseTools/Source/C/Makefiles/NmakeSubdirs.py b/BaseTools/Source/C/Makefiles/NmakeSubdirs.py new file mode 100644 index 0000000..29bb5df --- /dev/null +++ b/BaseTools/Source/C/Makefiles/NmakeSubdirs.py @@ -0,0 +1,169 @@ +# @file NmakeSubdirs.py +# This script support parallel build for nmake in windows environment. +# It supports Python2.x and Python3.x both. +# +# Copyright (c) 2018, Intel Corporation. All rights reserved.
+# +# This program and the accompanying materials +# are licensed and made available under the terms and conditions of the BSD License +# which accompanies this distribution. The full text of the license may be found at +# http://opensource.org/licenses/bsd-license.php +# +# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. +# + +# +# Import Modules +# + +from __future__ import print_function +import argparse +import threading +import time +import os +import subprocess +import multiprocessing +import copy +import sys +__prog__ = 'NmakeSubdirs' +__version__ = '%s Version %s' % (__prog__, '0.10 ') +__copyright__ = 'Copyright (c) 2018, Intel Corporation. All rights reserved.' +__description__ = 'Replace for NmakeSubdirs.bat in windows ,support parallel build for nmake.\n' + +cpu_count = multiprocessing.cpu_count() +output_lock = threading.Lock() +def RunCommand(WorkDir=None, *Args, **kwargs): + if WorkDir is None: + WorkDir = os.curdir + if "stderr" not in kwargs: + kwargs["stderr"] = subprocess.STDOUT + if "stdout" not in kwargs: + kwargs["stdout"] = subprocess.PIPE + p = subprocess.Popen(Args, cwd=WorkDir, stderr=kwargs["stderr"], stdout=kwargs["stdout"]) + stdout, stderr = p.communicate() + message = "" + if stdout is not None: + message = stdout.decode() #for compatibility in python 2 and 3 + + if p.returncode != 0: + raise RuntimeError("Error while execute command \'{0}\' in direcotry {1}\n{2}".format(" ".join(Args), WorkDir, message)) + + output_lock.acquire(True) + print("execute command \"{0}\" in directory {1}".format(" ".join(Args), WorkDir)) + print(message) + output_lock.release() + + return p.returncode, stdout + +class TaskUnit(object): + def __init__(self, func, args, kwargs): + self.func = func + self.args = args + self.kwargs = kwargs + + def __eq__(self, other): + return id(self).__eq__(id(other)) + + def run(self): + return self.func(*self.args, **self.kwargs) + + def __str__(self): + para = list(self.args) + para.extend("{0}={1}".format(k, v)for k, v in self.kwargs.items()) + + return "{0}({1})".format(self.func.__name__, ",".join(para)) + +class ThreadControl(object): + + def __init__(self, maxthread): + self._processNum = maxthread + self.pending = [] + self.running = [] + self.pendingLock = threading.Lock() + self.runningLock = threading.Lock() + self.error = False + self.errorLock = threading.Lock() + self.errorMsg = "errorMsg" + + def addTask(self, func, *args, **kwargs): + self.pending.append(TaskUnit(func, args, kwargs)) + + def waitComplete(self): + self._schedule.join() + + def startSchedule(self): + self._schedule = threading.Thread(target=self.Schedule) + self._schedule.start() + + def Schedule(self): + for i in range(self._processNum): + task = threading.Thread(target=self.startTask) + task.daemon = False + self.running.append(task) + + self.runningLock.acquire(True) + for thread in self.running: + thread.start() + self.runningLock.release() + + while len(self.running) > 0: + time.sleep(0.1) + if self.error: + print("subprocess not exit sucessfully") + print(self.errorMsg) + + def startTask(self): + while True: + if self.error: + break + self.pendingLock.acquire(True) + if len(self.pending) == 0: + self.pendingLock.release() + break + task = self.pending.pop(0) + self.pendingLock.release() + try: + task.run() + except RuntimeError as e: + if self.error: break + self.errorLock.acquire(True) + self.error = True + self.errorMsg = str(e) + time.sleep(0.1) + self.errorLock.release() + break + + self.runningLock.acquire(True) + self.running.remove(threading.currentThread()) + self.runningLock.release() + +def Run(): + curdir = os.path.abspath(os.curdir) + if len(args.subdirs) == 1: + args.jobs = 1 + if args.jobs == 1: + try: + for dir in args.subdirs: + RunCommand(os.path.join(curdir, dir), "nmake", args.target, stdout=sys.stdout, stderr=subprocess.STDOUT) + except RuntimeError: + exit(1) + else: + controller = ThreadControl(args.jobs) + for dir in args.subdirs: + controller.addTask(RunCommand, os.path.join(curdir, dir), "nmake", args.target) + controller.startSchedule() + controller.waitComplete() + if controller.error: + exit(1) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(prog=__prog__, description=__description__ + __copyright__, conflict_handler='resolve') + + parser.add_argument("target", help="the target for nmake") + parser.add_argument("subdirs", nargs="+", help="the relative dir path of makefile") + parser.add_argument("--jobs", type=int, dest="jobs", default=cpu_count, help="thread number") + parser.add_argument('--version', action='version', version=__version__) + args = parser.parse_args() + Run() + -- 2.10.0.windows.1