* [Patch] BaseTools:Enable the /MP option of MSVC compiler
@ 2019-03-29 12:30 Feng, Bob C
2019-03-29 14:20 ` Gao, Liming
2019-03-30 1:42 ` Ni, Ray
0 siblings, 2 replies; 8+ messages in thread
From: Feng, Bob C @ 2019-03-29 12:30 UTC (permalink / raw)
To: edk2-devel; +Cc: Zhiju Fan, Bob Feng, Liming Gao
From: Zhiju Fan <zhijux.fan@intel.com>
https://bugzilla.tianocore.org/show_bug.cgi?id=1672
The /MP option of MSVC compiler can reduce the total
time to compile the source files on the command line.
This patch is going to enable this MSVC option in BaseTools.
Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com>
---
BaseTools/Conf/build_rule.template | 2 +-
BaseTools/Source/Python/AutoGen/GenMake.py | 81 ++++++++++++++++++++--
2 files changed, 75 insertions(+), 8 deletions(-)
diff --git a/BaseTools/Conf/build_rule.template b/BaseTools/Conf/build_rule.template
index e56b1d9c59..e7d736740f 100755
--- a/BaseTools/Conf/build_rule.template
+++ b/BaseTools/Conf/build_rule.template
@@ -127,11 +127,11 @@
<OutputFile>
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj
<Command.MSFT, Command.INTEL>
- "$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src}
+ "$(CC)" /MP7 /Fo${d_path}\ $(CC_FLAGS) $(INC) ${src}
<Command.GCC, Command.RVCT>
# For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues
"$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src}
diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py
index b441817b52..04951346ad 100644
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -433,11 +433,11 @@ cleanlib:
self.BuildTargetList = [] # [target string]
self.PendingBuildTargetList = [] # [FileBuildRule objects]
self.CommonFileDependency = []
self.FileListMacros = {}
self.ListFileMacros = {}
-
+ self.ObjTargetDict = {}
self.FileCache = {}
self.LibraryBuildCommandList = []
self.LibraryFileList = []
self.LibraryMakefileList = []
self.LibraryBuildDirectoryList = []
@@ -616,10 +616,15 @@ cleanlib:
ListFileName,
"\n".join(self.ListFileMacros[ListFileMacro]),
False
)
+ # Generate objlist used to create .obj file
+ for Type in self.ObjTargetDict:
+ NewLine = ' '.join(list(self.ObjTargetDict[Type]))
+ FileMacroList.append("OBJLIST_%s = %s" % (list(self.ObjTargetDict.keys()).index(Type), NewLine))
+
BcTargetList = []
MakefileName = self._FILE_NAME_[self._FileType]
LibraryMakeCommandList = []
for D in self.LibraryBuildDirectoryList:
@@ -925,17 +930,22 @@ cleanlib:
# Extract common files list in the dependency files
#
for File in DepSet:
self.CommonFileDependency.append(self.PlaceMacro(File.Path, self.Macros))
+ CmdSumDict = {}
+ CmdTargetDict = {}
+ CmdCppDict = {}
+ DependencyDict = FileDependencyDict.copy()
for File in FileDependencyDict:
# skip non-C files
if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c":
continue
NewDepSet = set(FileDependencyDict[File])
NewDepSet -= DepSet
FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet)
+ DependencyDict[File] = list(NewDepSet)
# Convert target description object to target string in makefile
for Type in self._AutoGenObject.Targets:
for T in self._AutoGenObject.Targets[Type]:
# Generate related macros if needed
@@ -943,15 +953,25 @@ cleanlib:
self.FileListMacros[T.FileListMacro] = []
if T.GenListFile and T.ListFileMacro not in self.ListFileMacros:
self.ListFileMacros[T.ListFileMacro] = []
if T.GenIncListFile and T.IncListFileMacro not in self.ListFileMacros:
self.ListFileMacros[T.IncListFileMacro] = []
+ if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE:
+ NewFile = self.PlaceMacro(str(T), self.Macros)
+ if self.ObjTargetDict.get(T.Target.SubDir):
+ self.ObjTargetDict[T.Target.SubDir].add(NewFile)
+ else:
+ self.ObjTargetDict[T.Target.SubDir] = set()
+ self.ObjTargetDict[T.Target.SubDir].add(NewFile)
Deps = []
+ CCodeDeps = []
# Add force-dependencies
for Dep in T.Dependencies:
Deps.append(self.PlaceMacro(str(Dep), self.Macros))
+ if Dep != '$(MAKE_FILE)':
+ CCodeDeps.append(self.PlaceMacro(str(Dep), self.Macros))
# Add inclusion-dependencies
if len(T.Inputs) == 1 and T.Inputs[0] in FileDependencyDict:
for F in FileDependencyDict[T.Inputs[0]]:
Deps.append(self.PlaceMacro(str(F), self.Macros))
# Add source-dependencies
@@ -971,16 +991,63 @@ cleanlib:
if T.GenFileListMacro:
Deps.append("$(%s)" % T.FileListMacro)
if Type in [TAB_OBJECT_FILE, TAB_STATIC_LIBRARY]:
Deps.append("$(%s)" % T.ListFileMacro)
- TargetDict = {
- "target" : self.PlaceMacro(T.Target.Path, self.Macros),
- "cmd" : "\n\t".join(T.Commands),
- "deps" : Deps
- }
- self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDict))
+ if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE:
+ T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict)
+ TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps}
+ CmdLine = self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIST', '$(OBJLIST')
+ if T.Commands:
+ CmdLine = '%s%s' %(CmdLine, TAB_LINE_BREAK)
+ if CCodeDeps or CmdLine:
+ self.BuildTargetList.append(CmdLine)
+ else:
+ TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": Deps}
+ self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDict))
+
+ def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict):
+ if not CmdSumDict:
+ for item in self._AutoGenObject.Targets[Type]:
+ CmdSumDict[item.Target.SubDir] = item.Target.BaseName
+ for CppPath in item.Inputs:
+ Path = self.PlaceMacro(CppPath.Path, self.Macros)
+ if CmdCppDict.get(item.Target.SubDir):
+ CmdCppDict[item.Target.SubDir].append(Path)
+ else:
+ CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path]
+ if CppPath.Path in DependencyDict:
+ for Temp in DependencyDict[CppPath.Path]:
+ Path = self.PlaceMacro(Temp.Path, self.Macros)
+ if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]):
+ CmdCppDict[item.Target.SubDir].append(Path)
+ if T.Commands:
+ CommandList = T.Commands[:]
+ for Item in CommandList[:]:
+ SingleCommandList = Item.split()
+ if len(SingleCommandList) > 0 and '$(CC)' in SingleCommandList[0]:
+ for Temp in SingleCommandList:
+ if Temp.startswith('/Fo'):
+ CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH)
+ break
+ else: continue
+ if CmdSign not in list(CmdTargetDict.keys()):
+ CmdLine = Item.replace(Temp, CmdSign)
+ if GlobalData.gOptions.ThreadNumber:
+ CmdLine = CmdLine.replace('/MP7', '%s%d' % ('/MP', GlobalData.gOptions.ThreadNumber))
+ CmdTargetDict[CmdSign] = CmdLine
+ else:
+ CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1])
+ Index = CommandList.index(Item)
+ CommandList.pop(Index)
+ if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign.lstrip('/Fo').rsplit(TAB_SLASH, 1)[0]])):
+ Cpplist = CmdCppDict[T.Target.SubDir]
+ Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir))
+ T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign])
+ else:
+ T.Commands.pop(Index)
+ return T, CmdSumDict, CmdTargetDict, CmdCppDict
## For creating makefile targets for dependent libraries
def ProcessDependentLibrary(self):
for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList:
if not LibraryAutoGen.IsBinaryModule:
--
2.20.1.windows.1
^ permalink raw reply related [flat|nested] 8+ messages in thread
* Re: [Patch] BaseTools:Enable the /MP option of MSVC compiler
2019-03-29 12:30 [Patch] BaseTools:Enable the /MP option of MSVC compiler Feng, Bob C
@ 2019-03-29 14:20 ` Gao, Liming
2019-04-03 2:11 ` Feng, Bob C
2019-03-30 1:42 ` Ni, Ray
1 sibling, 1 reply; 8+ messages in thread
From: Gao, Liming @ 2019-03-29 14:20 UTC (permalink / raw)
To: Feng, Bob C, edk2-devel@lists.01.org
Bob:
How about use /MP option without process number? The compiler will retrieves the number of effective processors on your computer.
And, move this option into CC_FLAGS in tools_def.txt. If so, platform can override it in platform DSC file.
Last, please provide the performance data with this option.
Thanks
Liming
> -----Original Message-----
> From: Feng, Bob C
> Sent: Friday, March 29, 2019 8:30 PM
> To: edk2-devel@lists.01.org
> Cc: Fan, ZhijuX <zhijux.fan@intel.com>; Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming <liming.gao@intel.com>
> Subject: [Patch] BaseTools:Enable the /MP option of MSVC compiler
>
> From: Zhiju Fan <zhijux.fan@intel.com>
>
> https://bugzilla.tianocore.org/show_bug.cgi?id=1672
> The /MP option of MSVC compiler can reduce the total
> time to compile the source files on the command line.
>
> This patch is going to enable this MSVC option in BaseTools.
>
> Cc: Bob Feng <bob.c.feng@intel.com>
> Cc: Liming Gao <liming.gao@intel.com>
> Contributed-under: TianoCore Contribution Agreement 1.1
> Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com>
> ---
> BaseTools/Conf/build_rule.template | 2 +-
> BaseTools/Source/Python/AutoGen/GenMake.py | 81 ++++++++++++++++++++--
> 2 files changed, 75 insertions(+), 8 deletions(-)
>
> diff --git a/BaseTools/Conf/build_rule.template b/BaseTools/Conf/build_rule.template
> index e56b1d9c59..e7d736740f 100755
> --- a/BaseTools/Conf/build_rule.template
> +++ b/BaseTools/Conf/build_rule.template
> @@ -127,11 +127,11 @@
>
> <OutputFile>
> $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj
>
> <Command.MSFT, Command.INTEL>
> - "$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src}
> + "$(CC)" /MP7 /Fo${d_path}\ $(CC_FLAGS) $(INC) ${src}
>
> <Command.GCC, Command.RVCT>
> # For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues
> "$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src}
>
> diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py
> index b441817b52..04951346ad 100644
> --- a/BaseTools/Source/Python/AutoGen/GenMake.py
> +++ b/BaseTools/Source/Python/AutoGen/GenMake.py
> @@ -433,11 +433,11 @@ cleanlib:
> self.BuildTargetList = [] # [target string]
> self.PendingBuildTargetList = [] # [FileBuildRule objects]
> self.CommonFileDependency = []
> self.FileListMacros = {}
> self.ListFileMacros = {}
> -
> + self.ObjTargetDict = {}
> self.FileCache = {}
> self.LibraryBuildCommandList = []
> self.LibraryFileList = []
> self.LibraryMakefileList = []
> self.LibraryBuildDirectoryList = []
> @@ -616,10 +616,15 @@ cleanlib:
> ListFileName,
> "\n".join(self.ListFileMacros[ListFileMacro]),
> False
> )
>
> + # Generate objlist used to create .obj file
> + for Type in self.ObjTargetDict:
> + NewLine = ' '.join(list(self.ObjTargetDict[Type]))
> + FileMacroList.append("OBJLIST_%s = %s" % (list(self.ObjTargetDict.keys()).index(Type), NewLine))
> +
> BcTargetList = []
>
> MakefileName = self._FILE_NAME_[self._FileType]
> LibraryMakeCommandList = []
> for D in self.LibraryBuildDirectoryList:
> @@ -925,17 +930,22 @@ cleanlib:
> # Extract common files list in the dependency files
> #
> for File in DepSet:
> self.CommonFileDependency.append(self.PlaceMacro(File.Path, self.Macros))
>
> + CmdSumDict = {}
> + CmdTargetDict = {}
> + CmdCppDict = {}
> + DependencyDict = FileDependencyDict.copy()
> for File in FileDependencyDict:
> # skip non-C files
> if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c":
> continue
> NewDepSet = set(FileDependencyDict[File])
> NewDepSet -= DepSet
> FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet)
> + DependencyDict[File] = list(NewDepSet)
>
> # Convert target description object to target string in makefile
> for Type in self._AutoGenObject.Targets:
> for T in self._AutoGenObject.Targets[Type]:
> # Generate related macros if needed
> @@ -943,15 +953,25 @@ cleanlib:
> self.FileListMacros[T.FileListMacro] = []
> if T.GenListFile and T.ListFileMacro not in self.ListFileMacros:
> self.ListFileMacros[T.ListFileMacro] = []
> if T.GenIncListFile and T.IncListFileMacro not in self.ListFileMacros:
> self.ListFileMacros[T.IncListFileMacro] = []
> + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE:
> + NewFile = self.PlaceMacro(str(T), self.Macros)
> + if self.ObjTargetDict.get(T.Target.SubDir):
> + self.ObjTargetDict[T.Target.SubDir].add(NewFile)
> + else:
> + self.ObjTargetDict[T.Target.SubDir] = set()
> + self.ObjTargetDict[T.Target.SubDir].add(NewFile)
>
> Deps = []
> + CCodeDeps = []
> # Add force-dependencies
> for Dep in T.Dependencies:
> Deps.append(self.PlaceMacro(str(Dep), self.Macros))
> + if Dep != '$(MAKE_FILE)':
> + CCodeDeps.append(self.PlaceMacro(str(Dep), self.Macros))
> # Add inclusion-dependencies
> if len(T.Inputs) == 1 and T.Inputs[0] in FileDependencyDict:
> for F in FileDependencyDict[T.Inputs[0]]:
> Deps.append(self.PlaceMacro(str(F), self.Macros))
> # Add source-dependencies
> @@ -971,16 +991,63 @@ cleanlib:
> if T.GenFileListMacro:
> Deps.append("$(%s)" % T.FileListMacro)
> if Type in [TAB_OBJECT_FILE, TAB_STATIC_LIBRARY]:
> Deps.append("$(%s)" % T.ListFileMacro)
>
> - TargetDict = {
> - "target" : self.PlaceMacro(T.Target.Path, self.Macros),
> - "cmd" : "\n\t".join(T.Commands),
> - "deps" : Deps
> - }
> - self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDict))
> + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE:
> + T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict,
> CmdCppDict, DependencyDict)
> + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps":
> CCodeDeps}
> + CmdLine = self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIST', '$(OBJLIST')
> + if T.Commands:
> + CmdLine = '%s%s' %(CmdLine, TAB_LINE_BREAK)
> + if CCodeDeps or CmdLine:
> + self.BuildTargetList.append(CmdLine)
> + else:
> + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": Deps}
> + self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDict))
> +
> + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict):
> + if not CmdSumDict:
> + for item in self._AutoGenObject.Targets[Type]:
> + CmdSumDict[item.Target.SubDir] = item.Target.BaseName
> + for CppPath in item.Inputs:
> + Path = self.PlaceMacro(CppPath.Path, self.Macros)
> + if CmdCppDict.get(item.Target.SubDir):
> + CmdCppDict[item.Target.SubDir].append(Path)
> + else:
> + CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path]
> + if CppPath.Path in DependencyDict:
> + for Temp in DependencyDict[CppPath.Path]:
> + Path = self.PlaceMacro(Temp.Path, self.Macros)
> + if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]):
> + CmdCppDict[item.Target.SubDir].append(Path)
> + if T.Commands:
> + CommandList = T.Commands[:]
> + for Item in CommandList[:]:
> + SingleCommandList = Item.split()
> + if len(SingleCommandList) > 0 and '$(CC)' in SingleCommandList[0]:
> + for Temp in SingleCommandList:
> + if Temp.startswith('/Fo'):
> + CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH)
> + break
> + else: continue
> + if CmdSign not in list(CmdTargetDict.keys()):
> + CmdLine = Item.replace(Temp, CmdSign)
> + if GlobalData.gOptions.ThreadNumber:
> + CmdLine = CmdLine.replace('/MP7', '%s%d' % ('/MP', GlobalData.gOptions.ThreadNumber))
> + CmdTargetDict[CmdSign] = CmdLine
> + else:
> + CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1])
> + Index = CommandList.index(Item)
> + CommandList.pop(Index)
> + if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign.lstrip('/Fo').rsplit(TAB_SLASH,
> 1)[0]])):
> + Cpplist = CmdCppDict[T.Target.SubDir]
> + Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir))
> + T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign])
> + else:
> + T.Commands.pop(Index)
> + return T, CmdSumDict, CmdTargetDict, CmdCppDict
>
> ## For creating makefile targets for dependent libraries
> def ProcessDependentLibrary(self):
> for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList:
> if not LibraryAutoGen.IsBinaryModule:
> --
> 2.20.1.windows.1
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Patch] BaseTools:Enable the /MP option of MSVC compiler
2019-03-29 12:30 [Patch] BaseTools:Enable the /MP option of MSVC compiler Feng, Bob C
2019-03-29 14:20 ` Gao, Liming
@ 2019-03-30 1:42 ` Ni, Ray
2019-03-30 8:14 ` Minnow Ware
2019-03-30 11:19 ` Feng, Bob C
1 sibling, 2 replies; 8+ messages in thread
From: Ni, Ray @ 2019-03-30 1:42 UTC (permalink / raw)
To: Feng, Bob C, edk2-devel@lists.01.org; +Cc: Gao, Liming
https://devblogs.microsoft.com/cppblog/recommendations-to-speed-c-builds-in-visual-studio/#_PCH
Bob,
Per above article, It seems /Gm needs to be removed when /MP is used.
> -----Original Message-----
> From: edk2-devel [mailto:edk2-devel-bounces@lists.01.org] On Behalf Of Feng,
> Bob C
> Sent: Friday, March 29, 2019 8:30 PM
> To: edk2-devel@lists.01.org
> Cc: Gao, Liming <liming.gao@intel.com>
> Subject: [edk2] [Patch] BaseTools:Enable the /MP option of MSVC compiler
>
> From: Zhiju Fan <zhijux.fan@intel.com>
>
> https://bugzilla.tianocore.org/show_bug.cgi?id=1672
> The /MP option of MSVC compiler can reduce the total time to compile the
> source files on the command line.
>
> This patch is going to enable this MSVC option in BaseTools.
>
> Cc: Bob Feng <bob.c.feng@intel.com>
> Cc: Liming Gao <liming.gao@intel.com>
> Contributed-under: TianoCore Contribution Agreement 1.1
> Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com>
> ---
> BaseTools/Conf/build_rule.template | 2 +-
> BaseTools/Source/Python/AutoGen/GenMake.py | 81 ++++++++++++++++++++-
> -
> 2 files changed, 75 insertions(+), 8 deletions(-)
>
> diff --git a/BaseTools/Conf/build_rule.template
> b/BaseTools/Conf/build_rule.template
> index e56b1d9c59..e7d736740f 100755
> --- a/BaseTools/Conf/build_rule.template
> +++ b/BaseTools/Conf/build_rule.template
> @@ -127,11 +127,11 @@
>
> <OutputFile>
> $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj
>
> <Command.MSFT, Command.INTEL>
> - "$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src}
> + "$(CC)" /MP7 /Fo${d_path}\ $(CC_FLAGS) $(INC) ${src}
>
> <Command.GCC, Command.RVCT>
> # For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues
> "$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src}
>
> diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py
> b/BaseTools/Source/Python/AutoGen/GenMake.py
> index b441817b52..04951346ad 100644
> --- a/BaseTools/Source/Python/AutoGen/GenMake.py
> +++ b/BaseTools/Source/Python/AutoGen/GenMake.py
> @@ -433,11 +433,11 @@ cleanlib:
> self.BuildTargetList = [] # [target string]
> self.PendingBuildTargetList = [] # [FileBuildRule objects]
> self.CommonFileDependency = []
> self.FileListMacros = {}
> self.ListFileMacros = {}
> -
> + self.ObjTargetDict = {}
> self.FileCache = {}
> self.LibraryBuildCommandList = []
> self.LibraryFileList = []
> self.LibraryMakefileList = []
> self.LibraryBuildDirectoryList = [] @@ -616,10 +616,15 @@ cleanlib:
> ListFileName,
> "\n".join(self.ListFileMacros[ListFileMacro]),
> False
> )
>
> + # Generate objlist used to create .obj file
> + for Type in self.ObjTargetDict:
> + NewLine = ' '.join(list(self.ObjTargetDict[Type]))
> + FileMacroList.append("OBJLIST_%s = %s" %
> + (list(self.ObjTargetDict.keys()).index(Type), NewLine))
> +
> BcTargetList = []
>
> MakefileName = self._FILE_NAME_[self._FileType]
> LibraryMakeCommandList = []
> for D in self.LibraryBuildDirectoryList:
> @@ -925,17 +930,22 @@ cleanlib:
> # Extract common files list in the dependency files
> #
> for File in DepSet:
> self.CommonFileDependency.append(self.PlaceMacro(File.Path,
> self.Macros))
>
> + CmdSumDict = {}
> + CmdTargetDict = {}
> + CmdCppDict = {}
> + DependencyDict = FileDependencyDict.copy()
> for File in FileDependencyDict:
> # skip non-C files
> if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c":
> continue
> NewDepSet = set(FileDependencyDict[File])
> NewDepSet -= DepSet
> FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet)
> + DependencyDict[File] = list(NewDepSet)
>
> # Convert target description object to target string in makefile
> for Type in self._AutoGenObject.Targets:
> for T in self._AutoGenObject.Targets[Type]:
> # Generate related macros if needed @@ -943,15 +953,25 @@
> cleanlib:
> self.FileListMacros[T.FileListMacro] = []
> if T.GenListFile and T.ListFileMacro not in self.ListFileMacros:
> self.ListFileMacros[T.ListFileMacro] = []
> if T.GenIncListFile and T.IncListFileMacro not in self.ListFileMacros:
> self.ListFileMacros[T.IncListFileMacro] = []
> + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and
> Type == TAB_C_CODE_FILE:
> + NewFile = self.PlaceMacro(str(T), self.Macros)
> + if self.ObjTargetDict.get(T.Target.SubDir):
> + self.ObjTargetDict[T.Target.SubDir].add(NewFile)
> + else:
> + self.ObjTargetDict[T.Target.SubDir] = set()
> +
> + self.ObjTargetDict[T.Target.SubDir].add(NewFile)
>
> Deps = []
> + CCodeDeps = []
> # Add force-dependencies
> for Dep in T.Dependencies:
> Deps.append(self.PlaceMacro(str(Dep), self.Macros))
> + if Dep != '$(MAKE_FILE)':
> + CCodeDeps.append(self.PlaceMacro(str(Dep),
> + self.Macros))
> # Add inclusion-dependencies
> if len(T.Inputs) == 1 and T.Inputs[0] in FileDependencyDict:
> for F in FileDependencyDict[T.Inputs[0]]:
> Deps.append(self.PlaceMacro(str(F), self.Macros))
> # Add source-dependencies @@ -971,16 +991,63 @@ cleanlib:
> if T.GenFileListMacro:
> Deps.append("$(%s)" % T.FileListMacro)
> if Type in [TAB_OBJECT_FILE, TAB_STATIC_LIBRARY]:
> Deps.append("$(%s)" % T.ListFileMacro)
>
> - TargetDict = {
> - "target" : self.PlaceMacro(T.Target.Path, self.Macros),
> - "cmd" : "\n\t".join(T.Commands),
> - "deps" : Deps
> - }
> -
> self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDic
> t))
> + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and
> Type == TAB_C_CODE_FILE:
> + T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T,
> Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict)
> + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros),
> "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps}
> + CmdLine =
> self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIS
> T', '$(OBJLIST')
> + if T.Commands:
> + CmdLine = '%s%s' %(CmdLine, TAB_LINE_BREAK)
> + if CCodeDeps or CmdLine:
> + self.BuildTargetList.append(CmdLine)
> + else:
> + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros),
> "cmd": "\n\t".join(T.Commands),"deps": Deps}
> +
> + self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(Target
> + Dict))
> +
> + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict,
> DependencyDict):
> + if not CmdSumDict:
> + for item in self._AutoGenObject.Targets[Type]:
> + CmdSumDict[item.Target.SubDir] = item.Target.BaseName
> + for CppPath in item.Inputs:
> + Path = self.PlaceMacro(CppPath.Path, self.Macros)
> + if CmdCppDict.get(item.Target.SubDir):
> + CmdCppDict[item.Target.SubDir].append(Path)
> + else:
> + CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path]
> + if CppPath.Path in DependencyDict:
> + for Temp in DependencyDict[CppPath.Path]:
> + Path = self.PlaceMacro(Temp.Path, self.Macros)
> + if Path not in (self.CommonFileDependency +
> CmdCppDict[item.Target.SubDir]):
> + CmdCppDict[item.Target.SubDir].append(Path)
> + if T.Commands:
> + CommandList = T.Commands[:]
> + for Item in CommandList[:]:
> + SingleCommandList = Item.split()
> + if len(SingleCommandList) > 0 and '$(CC)' in SingleCommandList[0]:
> + for Temp in SingleCommandList:
> + if Temp.startswith('/Fo'):
> + CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH)
> + break
> + else: continue
> + if CmdSign not in list(CmdTargetDict.keys()):
> + CmdLine = Item.replace(Temp, CmdSign)
> + if GlobalData.gOptions.ThreadNumber:
> + CmdLine = CmdLine.replace('/MP7', '%s%d' % ('/MP',
> GlobalData.gOptions.ThreadNumber))
> + CmdTargetDict[CmdSign] = CmdLine
> + else:
> + CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign],
> SingleCommandList[-1])
> + Index = CommandList.index(Item)
> + CommandList.pop(Index)
> + if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH,
> CmdSumDict[CmdSign.lstrip('/Fo').rsplit(TAB_SLASH, 1)[0]])):
> + Cpplist = CmdCppDict[T.Target.SubDir]
> + Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' %
> list(self.ObjTargetDict.keys()).index(T.Target.SubDir))
> + T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist),
> CmdTargetDict[CmdSign])
> + else:
> + T.Commands.pop(Index)
> + return T, CmdSumDict, CmdTargetDict, CmdCppDict
>
> ## For creating makefile targets for dependent libraries
> def ProcessDependentLibrary(self):
> for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList:
> if not LibraryAutoGen.IsBinaryModule:
> --
> 2.20.1.windows.1
>
> _______________________________________________
> edk2-devel mailing list
> edk2-devel@lists.01.org
> https://lists.01.org/mailman/listinfo/edk2-devel
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Patch] BaseTools:Enable the /MP option of MSVC compiler
2019-03-30 1:42 ` Ni, Ray
@ 2019-03-30 8:14 ` Minnow Ware
2019-04-01 14:55 ` Feng, Bob C
2019-03-30 11:19 ` Feng, Bob C
1 sibling, 1 reply; 8+ messages in thread
From: Minnow Ware @ 2019-03-30 8:14 UTC (permalink / raw)
To: Ni, Ray, Feng, Bob C, edk2-devel@lists.01.org; +Cc: Gao, Liming
It would also be good, to add a REBUILD target to the makefiles,
that forces the compiler to translate all C-files from a component at once,
w/o dependency check e.g.:
REBUILD:
cl /c [additional switches] file1.c file2.c file3.c file4.c
so that the compiler itself is invoked only once per component.
This is possible for Microsoft, Intel and GNU C-compiler.
Best regards,
Kilian
https://github.com/MinnowWare
________________________________
From: edk2-devel <edk2-devel-bounces@lists.01.org> on behalf of Ni, Ray <ray.ni@intel.com>
Sent: Saturday, March 30, 2019 2:42:21 AM
To: Feng, Bob C; edk2-devel@lists.01.org
Cc: Gao, Liming
Subject: Re: [edk2] [Patch] BaseTools:Enable the /MP option of MSVC compiler
https://devblogs.microsoft.com/cppblog/recommendations-to-speed-c-builds-in-visual-studio/#_PCH
Bob,
Per above article, It seems /Gm needs to be removed when /MP is used.
> -----Original Message-----
> From: edk2-devel [mailto:edk2-devel-bounces@lists.01.org] On Behalf Of Feng,
> Bob C
> Sent: Friday, March 29, 2019 8:30 PM
> To: edk2-devel@lists.01.org
> Cc: Gao, Liming <liming.gao@intel.com>
> Subject: [edk2] [Patch] BaseTools:Enable the /MP option of MSVC compiler
>
> From: Zhiju Fan <zhijux.fan@intel.com>
>
> https://bugzilla.tianocore.org/show_bug.cgi?id=1672
> The /MP option of MSVC compiler can reduce the total time to compile the
> source files on the command line.
>
> This patch is going to enable this MSVC option in BaseTools.
>
> Cc: Bob Feng <bob.c.feng@intel.com>
> Cc: Liming Gao <liming.gao@intel.com>
> Contributed-under: TianoCore Contribution Agreement 1.1
> Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com>
> ---
> BaseTools/Conf/build_rule.template | 2 +-
> BaseTools/Source/Python/AutoGen/GenMake.py | 81 ++++++++++++++++++++-
> -
> 2 files changed, 75 insertions(+), 8 deletions(-)
>
> diff --git a/BaseTools/Conf/build_rule.template
> b/BaseTools/Conf/build_rule.template
> index e56b1d9c59..e7d736740f 100755
> --- a/BaseTools/Conf/build_rule.template
> +++ b/BaseTools/Conf/build_rule.template
> @@ -127,11 +127,11 @@
>
> <OutputFile>
> $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj
>
> <Command.MSFT, Command.INTEL>
> - "$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src}
> + "$(CC)" /MP7 /Fo${d_path}\ $(CC_FLAGS) $(INC) ${src}
>
> <Command.GCC, Command.RVCT>
> # For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues
> "$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src}
>
> diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py
> b/BaseTools/Source/Python/AutoGen/GenMake.py
> index b441817b52..04951346ad 100644
> --- a/BaseTools/Source/Python/AutoGen/GenMake.py
> +++ b/BaseTools/Source/Python/AutoGen/GenMake.py
> @@ -433,11 +433,11 @@ cleanlib:
> self.BuildTargetList = [] # [target string]
> self.PendingBuildTargetList = [] # [FileBuildRule objects]
> self.CommonFileDependency = []
> self.FileListMacros = {}
> self.ListFileMacros = {}
> -
> + self.ObjTargetDict = {}
> self.FileCache = {}
> self.LibraryBuildCommandList = []
> self.LibraryFileList = []
> self.LibraryMakefileList = []
> self.LibraryBuildDirectoryList = [] @@ -616,10 +616,15 @@ cleanlib:
> ListFileName,
> "\n".join(self.ListFileMacros[ListFileMacro]),
> False
> )
>
> + # Generate objlist used to create .obj file
> + for Type in self.ObjTargetDict:
> + NewLine = ' '.join(list(self.ObjTargetDict[Type]))
> + FileMacroList.append("OBJLIST_%s = %s" %
> + (list(self.ObjTargetDict.keys()).index(Type), NewLine))
> +
> BcTargetList = []
>
> MakefileName = self._FILE_NAME_[self._FileType]
> LibraryMakeCommandList = []
> for D in self.LibraryBuildDirectoryList:
> @@ -925,17 +930,22 @@ cleanlib:
> # Extract common files list in the dependency files
> #
> for File in DepSet:
> self.CommonFileDependency.append(self.PlaceMacro(File.Path,
> self.Macros))
>
> + CmdSumDict = {}
> + CmdTargetDict = {}
> + CmdCppDict = {}
> + DependencyDict = FileDependencyDict.copy()
> for File in FileDependencyDict:
> # skip non-C files
> if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c":
> continue
> NewDepSet = set(FileDependencyDict[File])
> NewDepSet -= DepSet
> FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet)
> + DependencyDict[File] = list(NewDepSet)
>
> # Convert target description object to target string in makefile
> for Type in self._AutoGenObject.Targets:
> for T in self._AutoGenObject.Targets[Type]:
> # Generate related macros if needed @@ -943,15 +953,25 @@
> cleanlib:
> self.FileListMacros[T.FileListMacro] = []
> if T.GenListFile and T.ListFileMacro not in self.ListFileMacros:
> self.ListFileMacros[T.ListFileMacro] = []
> if T.GenIncListFile and T.IncListFileMacro not in self.ListFileMacros:
> self.ListFileMacros[T.IncListFileMacro] = []
> + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and
> Type == TAB_C_CODE_FILE:
> + NewFile = self.PlaceMacro(str(T), self.Macros)
> + if self.ObjTargetDict.get(T.Target.SubDir):
> + self.ObjTargetDict[T.Target.SubDir].add(NewFile)
> + else:
> + self.ObjTargetDict[T.Target.SubDir] = set()
> +
> + self.ObjTargetDict[T.Target.SubDir].add(NewFile)
>
> Deps = []
> + CCodeDeps = []
> # Add force-dependencies
> for Dep in T.Dependencies:
> Deps.append(self.PlaceMacro(str(Dep), self.Macros))
> + if Dep != '$(MAKE_FILE)':
> + CCodeDeps.append(self.PlaceMacro(str(Dep),
> + self.Macros))
> # Add inclusion-dependencies
> if len(T.Inputs) == 1 and T.Inputs[0] in FileDependencyDict:
> for F in FileDependencyDict[T.Inputs[0]]:
> Deps.append(self.PlaceMacro(str(F), self.Macros))
> # Add source-dependencies @@ -971,16 +991,63 @@ cleanlib:
> if T.GenFileListMacro:
> Deps.append("$(%s)" % T.FileListMacro)
> if Type in [TAB_OBJECT_FILE, TAB_STATIC_LIBRARY]:
> Deps.append("$(%s)" % T.ListFileMacro)
>
> - TargetDict = {
> - "target" : self.PlaceMacro(T.Target.Path, self.Macros),
> - "cmd" : "\n\t".join(T.Commands),
> - "deps" : Deps
> - }
> -
> self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDic
> t))
> + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and
> Type == TAB_C_CODE_FILE:
> + T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T,
> Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict)
> + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros),
> "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps}
> + CmdLine =
> self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIS
> T', '$(OBJLIST')
> + if T.Commands:
> + CmdLine = '%s%s' %(CmdLine, TAB_LINE_BREAK)
> + if CCodeDeps or CmdLine:
> + self.BuildTargetList.append(CmdLine)
> + else:
> + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros),
> "cmd": "\n\t".join(T.Commands),"deps": Deps}
> +
> + self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(Target
> + Dict))
> +
> + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict,
> DependencyDict):
> + if not CmdSumDict:
> + for item in self._AutoGenObject.Targets[Type]:
> + CmdSumDict[item.Target.SubDir] = item.Target.BaseName
> + for CppPath in item.Inputs:
> + Path = self.PlaceMacro(CppPath.Path, self.Macros)
> + if CmdCppDict.get(item.Target.SubDir):
> + CmdCppDict[item.Target.SubDir].append(Path)
> + else:
> + CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path]
> + if CppPath.Path in DependencyDict:
> + for Temp in DependencyDict[CppPath.Path]:
> + Path = self.PlaceMacro(Temp.Path, self.Macros)
> + if Path not in (self.CommonFileDependency +
> CmdCppDict[item.Target.SubDir]):
> + CmdCppDict[item.Target.SubDir].append(Path)
> + if T.Commands:
> + CommandList = T.Commands[:]
> + for Item in CommandList[:]:
> + SingleCommandList = Item.split()
> + if len(SingleCommandList) > 0 and '$(CC)' in SingleCommandList[0]:
> + for Temp in SingleCommandList:
> + if Temp.startswith('/Fo'):
> + CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH)
> + break
> + else: continue
> + if CmdSign not in list(CmdTargetDict.keys()):
> + CmdLine = Item.replace(Temp, CmdSign)
> + if GlobalData.gOptions.ThreadNumber:
> + CmdLine = CmdLine.replace('/MP7', '%s%d' % ('/MP',
> GlobalData.gOptions.ThreadNumber))
> + CmdTargetDict[CmdSign] = CmdLine
> + else:
> + CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign],
> SingleCommandList[-1])
> + Index = CommandList.index(Item)
> + CommandList.pop(Index)
> + if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH,
> CmdSumDict[CmdSign.lstrip('/Fo').rsplit(TAB_SLASH, 1)[0]])):
> + Cpplist = CmdCppDict[T.Target.SubDir]
> + Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' %
> list(self.ObjTargetDict.keys()).index(T.Target.SubDir))
> + T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist),
> CmdTargetDict[CmdSign])
> + else:
> + T.Commands.pop(Index)
> + return T, CmdSumDict, CmdTargetDict, CmdCppDict
>
> ## For creating makefile targets for dependent libraries
> def ProcessDependentLibrary(self):
> for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList:
> if not LibraryAutoGen.IsBinaryModule:
> --
> 2.20.1.windows.1
>
> _______________________________________________
> edk2-devel mailing list
> edk2-devel@lists.01.org
> https://lists.01.org/mailman/listinfo/edk2-devel
_______________________________________________
edk2-devel mailing list
edk2-devel@lists.01.org
https://lists.01.org/mailman/listinfo/edk2-devel
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Patch] BaseTools:Enable the /MP option of MSVC compiler
2019-03-30 1:42 ` Ni, Ray
2019-03-30 8:14 ` Minnow Ware
@ 2019-03-30 11:19 ` Feng, Bob C
1 sibling, 0 replies; 8+ messages in thread
From: Feng, Bob C @ 2019-03-30 11:19 UTC (permalink / raw)
To: Ni, Ray, edk2-devel@lists.01.org; +Cc: Gao, Liming
Yes. Commit fb94f83131f032cd5ce027ea706c45513c1a799e removed the /Gm option from MSVC tool chain.
-----Original Message-----
From: Ni, Ray
Sent: Saturday, March 30, 2019 9:42 AM
To: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org
Cc: Gao, Liming <liming.gao@intel.com>
Subject: RE: [edk2] [Patch] BaseTools:Enable the /MP option of MSVC compiler
https://devblogs.microsoft.com/cppblog/recommendations-to-speed-c-builds-in-visual-studio/#_PCH
Bob,
Per above article, It seems /Gm needs to be removed when /MP is used.
> -----Original Message-----
> From: edk2-devel [mailto:edk2-devel-bounces@lists.01.org] On Behalf Of
> Feng, Bob C
> Sent: Friday, March 29, 2019 8:30 PM
> To: edk2-devel@lists.01.org
> Cc: Gao, Liming <liming.gao@intel.com>
> Subject: [edk2] [Patch] BaseTools:Enable the /MP option of MSVC
> compiler
>
> From: Zhiju Fan <zhijux.fan@intel.com>
>
> https://bugzilla.tianocore.org/show_bug.cgi?id=1672
> The /MP option of MSVC compiler can reduce the total time to compile
> the source files on the command line.
>
> This patch is going to enable this MSVC option in BaseTools.
>
> Cc: Bob Feng <bob.c.feng@intel.com>
> Cc: Liming Gao <liming.gao@intel.com>
> Contributed-under: TianoCore Contribution Agreement 1.1
> Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com>
> ---
> BaseTools/Conf/build_rule.template | 2 +-
> BaseTools/Source/Python/AutoGen/GenMake.py | 81 ++++++++++++++++++++-
> -
> 2 files changed, 75 insertions(+), 8 deletions(-)
>
> diff --git a/BaseTools/Conf/build_rule.template
> b/BaseTools/Conf/build_rule.template
> index e56b1d9c59..e7d736740f 100755
> --- a/BaseTools/Conf/build_rule.template
> +++ b/BaseTools/Conf/build_rule.template
> @@ -127,11 +127,11 @@
>
> <OutputFile>
> $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj
>
> <Command.MSFT, Command.INTEL>
> - "$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src}
> + "$(CC)" /MP7 /Fo${d_path}\ $(CC_FLAGS) $(INC) ${src}
>
> <Command.GCC, Command.RVCT>
> # For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues
> "$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src}
>
> diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py
> b/BaseTools/Source/Python/AutoGen/GenMake.py
> index b441817b52..04951346ad 100644
> --- a/BaseTools/Source/Python/AutoGen/GenMake.py
> +++ b/BaseTools/Source/Python/AutoGen/GenMake.py
> @@ -433,11 +433,11 @@ cleanlib:
> self.BuildTargetList = [] # [target string]
> self.PendingBuildTargetList = [] # [FileBuildRule objects]
> self.CommonFileDependency = []
> self.FileListMacros = {}
> self.ListFileMacros = {}
> -
> + self.ObjTargetDict = {}
> self.FileCache = {}
> self.LibraryBuildCommandList = []
> self.LibraryFileList = []
> self.LibraryMakefileList = []
> self.LibraryBuildDirectoryList = [] @@ -616,10 +616,15 @@ cleanlib:
> ListFileName,
> "\n".join(self.ListFileMacros[ListFileMacro]),
> False
> )
>
> + # Generate objlist used to create .obj file
> + for Type in self.ObjTargetDict:
> + NewLine = ' '.join(list(self.ObjTargetDict[Type]))
> + FileMacroList.append("OBJLIST_%s = %s" %
> + (list(self.ObjTargetDict.keys()).index(Type), NewLine))
> +
> BcTargetList = []
>
> MakefileName = self._FILE_NAME_[self._FileType]
> LibraryMakeCommandList = []
> for D in self.LibraryBuildDirectoryList:
> @@ -925,17 +930,22 @@ cleanlib:
> # Extract common files list in the dependency files
> #
> for File in DepSet:
>
> self.CommonFileDependency.append(self.PlaceMacro(File.Path,
> self.Macros))
>
> + CmdSumDict = {}
> + CmdTargetDict = {}
> + CmdCppDict = {}
> + DependencyDict = FileDependencyDict.copy()
> for File in FileDependencyDict:
> # skip non-C files
> if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c":
> continue
> NewDepSet = set(FileDependencyDict[File])
> NewDepSet -= DepSet
> FileDependencyDict[File] = ["$(COMMON_DEPS)"] +
> list(NewDepSet)
> + DependencyDict[File] = list(NewDepSet)
>
> # Convert target description object to target string in makefile
> for Type in self._AutoGenObject.Targets:
> for T in self._AutoGenObject.Targets[Type]:
> # Generate related macros if needed @@ -943,15
> +953,25 @@
> cleanlib:
> self.FileListMacros[T.FileListMacro] = []
> if T.GenListFile and T.ListFileMacro not in self.ListFileMacros:
> self.ListFileMacros[T.ListFileMacro] = []
> if T.GenIncListFile and T.IncListFileMacro not in self.ListFileMacros:
> self.ListFileMacros[T.IncListFileMacro] = []
> + if self._AutoGenObject.BuildRuleFamily ==
> + TAB_COMPILER_MSFT and
> Type == TAB_C_CODE_FILE:
> + NewFile = self.PlaceMacro(str(T), self.Macros)
> + if self.ObjTargetDict.get(T.Target.SubDir):
> + self.ObjTargetDict[T.Target.SubDir].add(NewFile)
> + else:
> + self.ObjTargetDict[T.Target.SubDir] = set()
> +
> + self.ObjTargetDict[T.Target.SubDir].add(NewFile)
>
> Deps = []
> + CCodeDeps = []
> # Add force-dependencies
> for Dep in T.Dependencies:
> Deps.append(self.PlaceMacro(str(Dep),
> self.Macros))
> + if Dep != '$(MAKE_FILE)':
> + CCodeDeps.append(self.PlaceMacro(str(Dep),
> + self.Macros))
> # Add inclusion-dependencies
> if len(T.Inputs) == 1 and T.Inputs[0] in FileDependencyDict:
> for F in FileDependencyDict[T.Inputs[0]]:
> Deps.append(self.PlaceMacro(str(F), self.Macros))
> # Add source-dependencies @@ -971,16 +991,63 @@ cleanlib:
> if T.GenFileListMacro:
> Deps.append("$(%s)" % T.FileListMacro)
> if Type in [TAB_OBJECT_FILE, TAB_STATIC_LIBRARY]:
> Deps.append("$(%s)" % T.ListFileMacro)
>
> - TargetDict = {
> - "target" : self.PlaceMacro(T.Target.Path, self.Macros),
> - "cmd" : "\n\t".join(T.Commands),
> - "deps" : Deps
> - }
> -
> self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(Target
> Dic
> t))
> + if self._AutoGenObject.BuildRuleFamily ==
> + TAB_COMPILER_MSFT and
> Type == TAB_C_CODE_FILE:
> + T, CmdTarget, CmdTargetDict, CmdCppDict =
> + self.ParserCCodeFile(T,
> Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict)
> + TargetDict = {"target":
> + self.PlaceMacro(T.Target.Path, self.Macros),
> "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps}
> + CmdLine =
> self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(
> OBJLIS
> T', '$(OBJLIST')
> + if T.Commands:
> + CmdLine = '%s%s' %(CmdLine, TAB_LINE_BREAK)
> + if CCodeDeps or CmdLine:
> + self.BuildTargetList.append(CmdLine)
> + else:
> + TargetDict = {"target":
> + self.PlaceMacro(T.Target.Path, self.Macros),
> "cmd": "\n\t".join(T.Commands),"deps": Deps}
> +
> + self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(Targ
> + et
> + Dict))
> +
> + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict,
> + CmdCppDict,
> DependencyDict):
> + if not CmdSumDict:
> + for item in self._AutoGenObject.Targets[Type]:
> + CmdSumDict[item.Target.SubDir] = item.Target.BaseName
> + for CppPath in item.Inputs:
> + Path = self.PlaceMacro(CppPath.Path, self.Macros)
> + if CmdCppDict.get(item.Target.SubDir):
> + CmdCppDict[item.Target.SubDir].append(Path)
> + else:
> + CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path]
> + if CppPath.Path in DependencyDict:
> + for Temp in DependencyDict[CppPath.Path]:
> + Path = self.PlaceMacro(Temp.Path, self.Macros)
> + if Path not in (self.CommonFileDependency
> + +
> CmdCppDict[item.Target.SubDir]):
> + CmdCppDict[item.Target.SubDir].append(Path)
> + if T.Commands:
> + CommandList = T.Commands[:]
> + for Item in CommandList[:]:
> + SingleCommandList = Item.split()
> + if len(SingleCommandList) > 0 and '$(CC)' in SingleCommandList[0]:
> + for Temp in SingleCommandList:
> + if Temp.startswith('/Fo'):
> + CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH)
> + break
> + else: continue
> + if CmdSign not in list(CmdTargetDict.keys()):
> + CmdLine = Item.replace(Temp, CmdSign)
> + if GlobalData.gOptions.ThreadNumber:
> + CmdLine = CmdLine.replace('/MP7', '%s%d'
> + % ('/MP',
> GlobalData.gOptions.ThreadNumber))
> + CmdTargetDict[CmdSign] = CmdLine
> + else:
> + CmdTargetDict[CmdSign] = "%s %s" %
> + (CmdTargetDict[CmdSign],
> SingleCommandList[-1])
> + Index = CommandList.index(Item)
> + CommandList.pop(Index)
> + if SingleCommandList[-1].endswith("%s%s.c" %
> + (TAB_SLASH,
> CmdSumDict[CmdSign.lstrip('/Fo').rsplit(TAB_SLASH, 1)[0]])):
> + Cpplist = CmdCppDict[T.Target.SubDir]
> + Cpplist.insert(0, '$(OBJLIST_%d):
> + $(COMMON_DEPS)' %
> list(self.ObjTargetDict.keys()).index(T.Target.SubDir))
> + T.Commands[Index] = '%s\n\t%s' % ('
> + \\\n\t'.join(Cpplist),
> CmdTargetDict[CmdSign])
> + else:
> + T.Commands.pop(Index)
> + return T, CmdSumDict, CmdTargetDict, CmdCppDict
>
> ## For creating makefile targets for dependent libraries
> def ProcessDependentLibrary(self):
> for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList:
> if not LibraryAutoGen.IsBinaryModule:
> --
> 2.20.1.windows.1
>
> _______________________________________________
> edk2-devel mailing list
> edk2-devel@lists.01.org
> https://lists.01.org/mailman/listinfo/edk2-devel
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Patch] BaseTools:Enable the /MP option of MSVC compiler
2019-03-30 8:14 ` Minnow Ware
@ 2019-04-01 14:55 ` Feng, Bob C
0 siblings, 0 replies; 8+ messages in thread
From: Feng, Bob C @ 2019-04-01 14:55 UTC (permalink / raw)
To: Minnow Ware, Ni, Ray, edk2-devel@lists.01.org; +Cc: Gao, Liming
Hi Kilian,
It's a good suggestion. Would you please enter a new BZ for this request? I think implement this request in a separate patch would be better.
Thanks,
Bob
From: Minnow Ware [mailto:minnowware@outlook.com]
Sent: Saturday, March 30, 2019 4:15 PM
To: Ni, Ray <ray.ni@intel.com>; Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org
Cc: Gao, Liming <liming.gao@intel.com>
Subject: RE: [edk2] [Patch] BaseTools:Enable the /MP option of MSVC compiler
It would also be good, to add a REBUILD target to the makefiles,
that forces the compiler to translate all C-files from a component at once,
w/o dependency check e.g.:
REBUILD:
cl /c [additional switches] file1.c file2.c file3.c file4.c
so that the compiler itself is invoked only once per component.
This is possible for Microsoft, Intel and GNU C-compiler.
Best regards,
Kilian
https://github.com/MinnowWare
________________________________
From: edk2-devel <edk2-devel-bounces@lists.01.org<mailto:edk2-devel-bounces@lists.01.org>> on behalf of Ni, Ray <ray.ni@intel.com<mailto:ray.ni@intel.com>>
Sent: Saturday, March 30, 2019 2:42:21 AM
To: Feng, Bob C; edk2-devel@lists.01.org<mailto:edk2-devel@lists.01.org>
Cc: Gao, Liming
Subject: Re: [edk2] [Patch] BaseTools:Enable the /MP option of MSVC compiler
https://devblogs.microsoft.com/cppblog/recommendations-to-speed-c-builds-in-visual-studio/#_PCH
Bob,
Per above article, It seems /Gm needs to be removed when /MP is used.
> -----Original Message-----
> From: edk2-devel [mailto:edk2-devel-bounces@lists.01.org] On Behalf Of Feng,
> Bob C
> Sent: Friday, March 29, 2019 8:30 PM
> To: edk2-devel@lists.01.org<mailto:edk2-devel@lists.01.org>
> Cc: Gao, Liming <liming.gao@intel.com<mailto:liming.gao@intel.com>>
> Subject: [edk2] [Patch] BaseTools:Enable the /MP option of MSVC compiler
>
> From: Zhiju Fan <zhijux.fan@intel.com<mailto:zhijux.fan@intel.com>>
>
> https://bugzilla.tianocore.org/show_bug.cgi?id=1672
> The /MP option of MSVC compiler can reduce the total time to compile the
> source files on the command line.
>
> This patch is going to enable this MSVC option in BaseTools.
>
> Cc: Bob Feng <bob.c.feng@intel.com<mailto:bob.c.feng@intel.com>>
> Cc: Liming Gao <liming.gao@intel.com<mailto:liming.gao@intel.com>>
> Contributed-under: TianoCore Contribution Agreement 1.1
> Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com<mailto:zhijux.fan@intel.com>>
> ---
> BaseTools/Conf/build_rule.template | 2 +-
> BaseTools/Source/Python/AutoGen/GenMake.py | 81 ++++++++++++++++++++-
> -
> 2 files changed, 75 insertions(+), 8 deletions(-)
>
> diff --git a/BaseTools/Conf/build_rule.template
> b/BaseTools/Conf/build_rule.template
> index e56b1d9c59..e7d736740f 100755
> --- a/BaseTools/Conf/build_rule.template
> +++ b/BaseTools/Conf/build_rule.template
> @@ -127,11 +127,11 @@
>
> <OutputFile>
> $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj
>
> <Command.MSFT, Command.INTEL>
> - "$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src}
> + "$(CC)" /MP7 /Fo${d_path}\ $(CC_FLAGS) $(INC) ${src}
>
> <Command.GCC, Command.RVCT>
> # For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues
> "$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src}
>
> diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py
> b/BaseTools/Source/Python/AutoGen/GenMake.py
> index b441817b52..04951346ad 100644
> --- a/BaseTools/Source/Python/AutoGen/GenMake.py
> +++ b/BaseTools/Source/Python/AutoGen/GenMake.py
> @@ -433,11 +433,11 @@ cleanlib:
> self.BuildTargetList = [] # [target string]
> self.PendingBuildTargetList = [] # [FileBuildRule objects]
> self.CommonFileDependency = []
> self.FileListMacros = {}
> self.ListFileMacros = {}
> -
> + self.ObjTargetDict = {}
> self.FileCache = {}
> self.LibraryBuildCommandList = []
> self.LibraryFileList = []
> self.LibraryMakefileList = []
> self.LibraryBuildDirectoryList = [] @@ -616,10 +616,15 @@ cleanlib:
> ListFileName,
> "\n".join(self.ListFileMacros[ListFileMacro]),
> False
> )
>
> + # Generate objlist used to create .obj file
> + for Type in self.ObjTargetDict:
> + NewLine = ' '.join(list(self.ObjTargetDict[Type]))
> + FileMacroList.append("OBJLIST_%s = %s" %
> + (list(self.ObjTargetDict.keys()).index(Type), NewLine))
> +
> BcTargetList = []
>
> MakefileName = self._FILE_NAME_[self._FileType]
> LibraryMakeCommandList = []
> for D in self.LibraryBuildDirectoryList:
> @@ -925,17 +930,22 @@ cleanlib:
> # Extract common files list in the dependency files
> #
> for File in DepSet:
> self.CommonFileDependency.append(self.PlaceMacro(File.Path,
> self.Macros))
>
> + CmdSumDict = {}
> + CmdTargetDict = {}
> + CmdCppDict = {}
> + DependencyDict = FileDependencyDict.copy()
> for File in FileDependencyDict:
> # skip non-C files
> if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c":
> continue
> NewDepSet = set(FileDependencyDict[File])
> NewDepSet -= DepSet
> FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet)
> + DependencyDict[File] = list(NewDepSet)
>
> # Convert target description object to target string in makefile
> for Type in self._AutoGenObject.Targets:
> for T in self._AutoGenObject.Targets[Type]:
> # Generate related macros if needed @@ -943,15 +953,25 @@
> cleanlib:
> self.FileListMacros[T.FileListMacro] = []
> if T.GenListFile and T.ListFileMacro not in self.ListFileMacros:
> self.ListFileMacros[T.ListFileMacro] = []
> if T.GenIncListFile and T.IncListFileMacro not in self.ListFileMacros:
> self.ListFileMacros[T.IncListFileMacro] = []
> + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and
> Type == TAB_C_CODE_FILE:
> + NewFile = self.PlaceMacro(str(T), self.Macros)
> + if self.ObjTargetDict.get(T.Target.SubDir):
> + self.ObjTargetDict[T.Target.SubDir].add(NewFile)
> + else:
> + self.ObjTargetDict[T.Target.SubDir] = set()
> +
> + self.ObjTargetDict[T.Target.SubDir].add(NewFile)
>
> Deps = []
> + CCodeDeps = []
> # Add force-dependencies
> for Dep in T.Dependencies:
> Deps.append(self.PlaceMacro(str(Dep), self.Macros))
> + if Dep != '$(MAKE_FILE)':
> + CCodeDeps.append(self.PlaceMacro(str(Dep),
> + self.Macros))
> # Add inclusion-dependencies
> if len(T.Inputs) == 1 and T.Inputs[0] in FileDependencyDict:
> for F in FileDependencyDict[T.Inputs[0]]:
> Deps.append(self.PlaceMacro(str(F), self.Macros))
> # Add source-dependencies @@ -971,16 +991,63 @@ cleanlib:
> if T.GenFileListMacro:
> Deps.append("$(%s)" % T.FileListMacro)
> if Type in [TAB_OBJECT_FILE, TAB_STATIC_LIBRARY]:
> Deps.append("$(%s)" % T.ListFileMacro)
>
> - TargetDict = {
> - "target" : self.PlaceMacro(T.Target.Path, self.Macros),
> - "cmd" : "\n\t".join(T.Commands),
> - "deps" : Deps
> - }
> -
> self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDic
> t))
> + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and
> Type == TAB_C_CODE_FILE:
> + T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T,
> Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict)
> + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros),
> "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps}
> + CmdLine =
> self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIS
> T', '$(OBJLIST')
> + if T.Commands:
> + CmdLine = '%s%s' %(CmdLine, TAB_LINE_BREAK)
> + if CCodeDeps or CmdLine:
> + self.BuildTargetList.append(CmdLine)
> + else:
> + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros),
> "cmd": "\n\t".join(T.Commands),"deps": Deps}
> +
> + self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(Target
> + Dict))
> +
> + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict,
> DependencyDict):
> + if not CmdSumDict:
> + for item in self._AutoGenObject.Targets[Type]:
> + CmdSumDict[item.Target.SubDir] = item.Target.BaseName
> + for CppPath in item.Inputs:
> + Path = self.PlaceMacro(CppPath.Path, self.Macros)
> + if CmdCppDict.get(item.Target.SubDir):
> + CmdCppDict[item.Target.SubDir].append(Path)
> + else:
> + CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path]
> + if CppPath.Path in DependencyDict:
> + for Temp in DependencyDict[CppPath.Path]:
> + Path = self.PlaceMacro(Temp.Path, self.Macros)
> + if Path not in (self.CommonFileDependency +
> CmdCppDict[item.Target.SubDir]):
> + CmdCppDict[item.Target.SubDir].append(Path)
> + if T.Commands:
> + CommandList = T.Commands[:]
> + for Item in CommandList[:]:
> + SingleCommandList = Item.split()
> + if len(SingleCommandList) > 0 and '$(CC)' in SingleCommandList[0]:
> + for Temp in SingleCommandList:
> + if Temp.startswith('/Fo'):
> + CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH)
> + break
> + else: continue
> + if CmdSign not in list(CmdTargetDict.keys()):
> + CmdLine = Item.replace(Temp, CmdSign)
> + if GlobalData.gOptions.ThreadNumber:
> + CmdLine = CmdLine.replace('/MP7', '%s%d' % ('/MP',
> GlobalData.gOptions.ThreadNumber))
> + CmdTargetDict[CmdSign] = CmdLine
> + else:
> + CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign],
> SingleCommandList[-1])
> + Index = CommandList.index(Item)
> + CommandList.pop(Index)
> + if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH,
> CmdSumDict[CmdSign.lstrip('/Fo').rsplit(TAB_SLASH, 1)[0]])):
> + Cpplist = CmdCppDict[T.Target.SubDir]
> + Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' %
> list(self.ObjTargetDict.keys()).index(T.Target.SubDir))
> + T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist)<file:///\\\n\t'.join(Cpplist)>,
> CmdTargetDict[CmdSign])
> + else:
> + T.Commands.pop(Index)
> + return T, CmdSumDict, CmdTargetDict, CmdCppDict
>
> ## For creating makefile targets for dependent libraries
> def ProcessDependentLibrary(self):
> for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList:
> if not LibraryAutoGen.IsBinaryModule:
> --
> 2.20.1.windows.1
>
> _______________________________________________
> edk2-devel mailing list
> edk2-devel@lists.01.org<mailto:edk2-devel@lists.01.org>
> https://lists.01.org/mailman/listinfo/edk2-devel
_______________________________________________
edk2-devel mailing list
edk2-devel@lists.01.org<mailto:edk2-devel@lists.01.org>
https://lists.01.org/mailman/listinfo/edk2-devel
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Patch] BaseTools:Enable the /MP option of MSVC compiler
2019-03-29 14:20 ` Gao, Liming
@ 2019-04-03 2:11 ` Feng, Bob C
2019-04-03 2:45 ` Gao, Liming
0 siblings, 1 reply; 8+ messages in thread
From: Feng, Bob C @ 2019-04-03 2:11 UTC (permalink / raw)
To: Gao, Liming, edk2-devel@lists.01.org
Liming,
I add the performance data on the BZ:
https://bugzilla.tianocore.org/show_bug.cgi?id=1672
Thanks,
Bob
-----Original Message-----
From: Gao, Liming
Sent: Friday, March 29, 2019 10:20 PM
To: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org
Cc: Fan, ZhijuX <zhijux.fan@intel.com>
Subject: RE: [Patch] BaseTools:Enable the /MP option of MSVC compiler
Bob:
How about use /MP option without process number? The compiler will retrieves the number of effective processors on your computer.
And, move this option into CC_FLAGS in tools_def.txt. If so, platform can override it in platform DSC file.
Last, please provide the performance data with this option.
Thanks
Liming
> -----Original Message-----
> From: Feng, Bob C
> Sent: Friday, March 29, 2019 8:30 PM
> To: edk2-devel@lists.01.org
> Cc: Fan, ZhijuX <zhijux.fan@intel.com>; Feng, Bob C
> <bob.c.feng@intel.com>; Gao, Liming <liming.gao@intel.com>
> Subject: [Patch] BaseTools:Enable the /MP option of MSVC compiler
>
> From: Zhiju Fan <zhijux.fan@intel.com>
>
> https://bugzilla.tianocore.org/show_bug.cgi?id=1672
> The /MP option of MSVC compiler can reduce the total time to compile
> the source files on the command line.
>
> This patch is going to enable this MSVC option in BaseTools.
>
> Cc: Bob Feng <bob.c.feng@intel.com>
> Cc: Liming Gao <liming.gao@intel.com>
> Contributed-under: TianoCore Contribution Agreement 1.1
> Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com>
> ---
> BaseTools/Conf/build_rule.template | 2 +-
> BaseTools/Source/Python/AutoGen/GenMake.py | 81
> ++++++++++++++++++++--
> 2 files changed, 75 insertions(+), 8 deletions(-)
>
> diff --git a/BaseTools/Conf/build_rule.template
> b/BaseTools/Conf/build_rule.template
> index e56b1d9c59..e7d736740f 100755
> --- a/BaseTools/Conf/build_rule.template
> +++ b/BaseTools/Conf/build_rule.template
> @@ -127,11 +127,11 @@
>
> <OutputFile>
> $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj
>
> <Command.MSFT, Command.INTEL>
> - "$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src}
> + "$(CC)" /MP7 /Fo${d_path}\ $(CC_FLAGS) $(INC) ${src}
>
> <Command.GCC, Command.RVCT>
> # For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues
> "$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src}
>
> diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py
> b/BaseTools/Source/Python/AutoGen/GenMake.py
> index b441817b52..04951346ad 100644
> --- a/BaseTools/Source/Python/AutoGen/GenMake.py
> +++ b/BaseTools/Source/Python/AutoGen/GenMake.py
> @@ -433,11 +433,11 @@ cleanlib:
> self.BuildTargetList = [] # [target string]
> self.PendingBuildTargetList = [] # [FileBuildRule objects]
> self.CommonFileDependency = []
> self.FileListMacros = {}
> self.ListFileMacros = {}
> -
> + self.ObjTargetDict = {}
> self.FileCache = {}
> self.LibraryBuildCommandList = []
> self.LibraryFileList = []
> self.LibraryMakefileList = []
> self.LibraryBuildDirectoryList = [] @@ -616,10 +616,15 @@
> cleanlib:
> ListFileName,
> "\n".join(self.ListFileMacros[ListFileMacro]),
> False
> )
>
> + # Generate objlist used to create .obj file
> + for Type in self.ObjTargetDict:
> + NewLine = ' '.join(list(self.ObjTargetDict[Type]))
> + FileMacroList.append("OBJLIST_%s = %s" %
> + (list(self.ObjTargetDict.keys()).index(Type), NewLine))
> +
> BcTargetList = []
>
> MakefileName = self._FILE_NAME_[self._FileType]
> LibraryMakeCommandList = []
> for D in self.LibraryBuildDirectoryList:
> @@ -925,17 +930,22 @@ cleanlib:
> # Extract common files list in the dependency files
> #
> for File in DepSet:
>
> self.CommonFileDependency.append(self.PlaceMacro(File.Path,
> self.Macros))
>
> + CmdSumDict = {}
> + CmdTargetDict = {}
> + CmdCppDict = {}
> + DependencyDict = FileDependencyDict.copy()
> for File in FileDependencyDict:
> # skip non-C files
> if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c":
> continue
> NewDepSet = set(FileDependencyDict[File])
> NewDepSet -= DepSet
> FileDependencyDict[File] = ["$(COMMON_DEPS)"] +
> list(NewDepSet)
> + DependencyDict[File] = list(NewDepSet)
>
> # Convert target description object to target string in makefile
> for Type in self._AutoGenObject.Targets:
> for T in self._AutoGenObject.Targets[Type]:
> # Generate related macros if needed @@ -943,15
> +953,25 @@ cleanlib:
> self.FileListMacros[T.FileListMacro] = []
> if T.GenListFile and T.ListFileMacro not in self.ListFileMacros:
> self.ListFileMacros[T.ListFileMacro] = []
> if T.GenIncListFile and T.IncListFileMacro not in self.ListFileMacros:
> self.ListFileMacros[T.IncListFileMacro] = []
> + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE:
> + NewFile = self.PlaceMacro(str(T), self.Macros)
> + if self.ObjTargetDict.get(T.Target.SubDir):
> + self.ObjTargetDict[T.Target.SubDir].add(NewFile)
> + else:
> + self.ObjTargetDict[T.Target.SubDir] = set()
> +
> + self.ObjTargetDict[T.Target.SubDir].add(NewFile)
>
> Deps = []
> + CCodeDeps = []
> # Add force-dependencies
> for Dep in T.Dependencies:
> Deps.append(self.PlaceMacro(str(Dep),
> self.Macros))
> + if Dep != '$(MAKE_FILE)':
> + CCodeDeps.append(self.PlaceMacro(str(Dep),
> + self.Macros))
> # Add inclusion-dependencies
> if len(T.Inputs) == 1 and T.Inputs[0] in FileDependencyDict:
> for F in FileDependencyDict[T.Inputs[0]]:
> Deps.append(self.PlaceMacro(str(F), self.Macros))
> # Add source-dependencies @@ -971,16 +991,63 @@
> cleanlib:
> if T.GenFileListMacro:
> Deps.append("$(%s)" % T.FileListMacro)
> if Type in [TAB_OBJECT_FILE, TAB_STATIC_LIBRARY]:
> Deps.append("$(%s)" % T.ListFileMacro)
>
> - TargetDict = {
> - "target" : self.PlaceMacro(T.Target.Path, self.Macros),
> - "cmd" : "\n\t".join(T.Commands),
> - "deps" : Deps
> - }
> - self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDict))
> + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE:
> + T, CmdTarget, CmdTargetDict, CmdCppDict =
> + self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict,
> CmdCppDict, DependencyDict)
> + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps":
> CCodeDeps}
> + CmdLine = self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIST', '$(OBJLIST')
> + if T.Commands:
> + CmdLine = '%s%s' %(CmdLine, TAB_LINE_BREAK)
> + if CCodeDeps or CmdLine:
> + self.BuildTargetList.append(CmdLine)
> + else:
> + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": Deps}
> +
> + self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(Targ
> + etDict))
> +
> + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict):
> + if not CmdSumDict:
> + for item in self._AutoGenObject.Targets[Type]:
> + CmdSumDict[item.Target.SubDir] = item.Target.BaseName
> + for CppPath in item.Inputs:
> + Path = self.PlaceMacro(CppPath.Path, self.Macros)
> + if CmdCppDict.get(item.Target.SubDir):
> + CmdCppDict[item.Target.SubDir].append(Path)
> + else:
> + CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path]
> + if CppPath.Path in DependencyDict:
> + for Temp in DependencyDict[CppPath.Path]:
> + Path = self.PlaceMacro(Temp.Path, self.Macros)
> + if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]):
> + CmdCppDict[item.Target.SubDir].append(Path)
> + if T.Commands:
> + CommandList = T.Commands[:]
> + for Item in CommandList[:]:
> + SingleCommandList = Item.split()
> + if len(SingleCommandList) > 0 and '$(CC)' in SingleCommandList[0]:
> + for Temp in SingleCommandList:
> + if Temp.startswith('/Fo'):
> + CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH)
> + break
> + else: continue
> + if CmdSign not in list(CmdTargetDict.keys()):
> + CmdLine = Item.replace(Temp, CmdSign)
> + if GlobalData.gOptions.ThreadNumber:
> + CmdLine = CmdLine.replace('/MP7', '%s%d' % ('/MP', GlobalData.gOptions.ThreadNumber))
> + CmdTargetDict[CmdSign] = CmdLine
> + else:
> + CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1])
> + Index = CommandList.index(Item)
> + CommandList.pop(Index)
> + if SingleCommandList[-1].endswith("%s%s.c" %
> + (TAB_SLASH, CmdSumDict[CmdSign.lstrip('/Fo').rsplit(TAB_SLASH,
> 1)[0]])):
> + Cpplist = CmdCppDict[T.Target.SubDir]
> + Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir))
> + T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign])
> + else:
> + T.Commands.pop(Index)
> + return T, CmdSumDict, CmdTargetDict, CmdCppDict
>
> ## For creating makefile targets for dependent libraries
> def ProcessDependentLibrary(self):
> for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList:
> if not LibraryAutoGen.IsBinaryModule:
> --
> 2.20.1.windows.1
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Patch] BaseTools:Enable the /MP option of MSVC compiler
2019-04-03 2:11 ` Feng, Bob C
@ 2019-04-03 2:45 ` Gao, Liming
0 siblings, 0 replies; 8+ messages in thread
From: Gao, Liming @ 2019-04-03 2:45 UTC (permalink / raw)
To: Feng, Bob C, edk2-devel@lists.01.org
The clean build performance data is good.
This change updates build rule generation for C source files on MSFT tool chain. It is a base tool behavior change. If it doesn't depend on build rule, could you drop the change in build rule.txt. MP flag will be moved to tools_def.txt.
Thanks
Liming
> -----Original Message-----
> From: Feng, Bob C
> Sent: Wednesday, April 3, 2019 10:12 AM
> To: Gao, Liming <liming.gao@intel.com>; edk2-devel@lists.01.org
> Cc: Fan, ZhijuX <zhijux.fan@intel.com>
> Subject: RE: [Patch] BaseTools:Enable the /MP option of MSVC compiler
>
> Liming,
>
> I add the performance data on the BZ:
> https://bugzilla.tianocore.org/show_bug.cgi?id=1672
>
> Thanks,
> Bob
>
> -----Original Message-----
> From: Gao, Liming
> Sent: Friday, March 29, 2019 10:20 PM
> To: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org
> Cc: Fan, ZhijuX <zhijux.fan@intel.com>
> Subject: RE: [Patch] BaseTools:Enable the /MP option of MSVC compiler
>
> Bob:
> How about use /MP option without process number? The compiler will retrieves the number of effective processors on your computer.
> And, move this option into CC_FLAGS in tools_def.txt. If so, platform can override it in platform DSC file.
>
> Last, please provide the performance data with this option.
>
> Thanks
> Liming
> > -----Original Message-----
> > From: Feng, Bob C
> > Sent: Friday, March 29, 2019 8:30 PM
> > To: edk2-devel@lists.01.org
> > Cc: Fan, ZhijuX <zhijux.fan@intel.com>; Feng, Bob C
> > <bob.c.feng@intel.com>; Gao, Liming <liming.gao@intel.com>
> > Subject: [Patch] BaseTools:Enable the /MP option of MSVC compiler
> >
> > From: Zhiju Fan <zhijux.fan@intel.com>
> >
> > https://bugzilla.tianocore.org/show_bug.cgi?id=1672
> > The /MP option of MSVC compiler can reduce the total time to compile
> > the source files on the command line.
> >
> > This patch is going to enable this MSVC option in BaseTools.
> >
> > Cc: Bob Feng <bob.c.feng@intel.com>
> > Cc: Liming Gao <liming.gao@intel.com>
> > Contributed-under: TianoCore Contribution Agreement 1.1
> > Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com>
> > ---
> > BaseTools/Conf/build_rule.template | 2 +-
> > BaseTools/Source/Python/AutoGen/GenMake.py | 81
> > ++++++++++++++++++++--
> > 2 files changed, 75 insertions(+), 8 deletions(-)
> >
> > diff --git a/BaseTools/Conf/build_rule.template
> > b/BaseTools/Conf/build_rule.template
> > index e56b1d9c59..e7d736740f 100755
> > --- a/BaseTools/Conf/build_rule.template
> > +++ b/BaseTools/Conf/build_rule.template
> > @@ -127,11 +127,11 @@
> >
> > <OutputFile>
> > $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj
> >
> > <Command.MSFT, Command.INTEL>
> > - "$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src}
> > + "$(CC)" /MP7 /Fo${d_path}\ $(CC_FLAGS) $(INC) ${src}
> >
> > <Command.GCC, Command.RVCT>
> > # For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues
> > "$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src}
> >
> > diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py
> > b/BaseTools/Source/Python/AutoGen/GenMake.py
> > index b441817b52..04951346ad 100644
> > --- a/BaseTools/Source/Python/AutoGen/GenMake.py
> > +++ b/BaseTools/Source/Python/AutoGen/GenMake.py
> > @@ -433,11 +433,11 @@ cleanlib:
> > self.BuildTargetList = [] # [target string]
> > self.PendingBuildTargetList = [] # [FileBuildRule objects]
> > self.CommonFileDependency = []
> > self.FileListMacros = {}
> > self.ListFileMacros = {}
> > -
> > + self.ObjTargetDict = {}
> > self.FileCache = {}
> > self.LibraryBuildCommandList = []
> > self.LibraryFileList = []
> > self.LibraryMakefileList = []
> > self.LibraryBuildDirectoryList = [] @@ -616,10 +616,15 @@
> > cleanlib:
> > ListFileName,
> > "\n".join(self.ListFileMacros[ListFileMacro]),
> > False
> > )
> >
> > + # Generate objlist used to create .obj file
> > + for Type in self.ObjTargetDict:
> > + NewLine = ' '.join(list(self.ObjTargetDict[Type]))
> > + FileMacroList.append("OBJLIST_%s = %s" %
> > + (list(self.ObjTargetDict.keys()).index(Type), NewLine))
> > +
> > BcTargetList = []
> >
> > MakefileName = self._FILE_NAME_[self._FileType]
> > LibraryMakeCommandList = []
> > for D in self.LibraryBuildDirectoryList:
> > @@ -925,17 +930,22 @@ cleanlib:
> > # Extract common files list in the dependency files
> > #
> > for File in DepSet:
> >
> > self.CommonFileDependency.append(self.PlaceMacro(File.Path,
> > self.Macros))
> >
> > + CmdSumDict = {}
> > + CmdTargetDict = {}
> > + CmdCppDict = {}
> > + DependencyDict = FileDependencyDict.copy()
> > for File in FileDependencyDict:
> > # skip non-C files
> > if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c":
> > continue
> > NewDepSet = set(FileDependencyDict[File])
> > NewDepSet -= DepSet
> > FileDependencyDict[File] = ["$(COMMON_DEPS)"] +
> > list(NewDepSet)
> > + DependencyDict[File] = list(NewDepSet)
> >
> > # Convert target description object to target string in makefile
> > for Type in self._AutoGenObject.Targets:
> > for T in self._AutoGenObject.Targets[Type]:
> > # Generate related macros if needed @@ -943,15
> > +953,25 @@ cleanlib:
> > self.FileListMacros[T.FileListMacro] = []
> > if T.GenListFile and T.ListFileMacro not in self.ListFileMacros:
> > self.ListFileMacros[T.ListFileMacro] = []
> > if T.GenIncListFile and T.IncListFileMacro not in self.ListFileMacros:
> > self.ListFileMacros[T.IncListFileMacro] = []
> > + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE:
> > + NewFile = self.PlaceMacro(str(T), self.Macros)
> > + if self.ObjTargetDict.get(T.Target.SubDir):
> > + self.ObjTargetDict[T.Target.SubDir].add(NewFile)
> > + else:
> > + self.ObjTargetDict[T.Target.SubDir] = set()
> > +
> > + self.ObjTargetDict[T.Target.SubDir].add(NewFile)
> >
> > Deps = []
> > + CCodeDeps = []
> > # Add force-dependencies
> > for Dep in T.Dependencies:
> > Deps.append(self.PlaceMacro(str(Dep),
> > self.Macros))
> > + if Dep != '$(MAKE_FILE)':
> > + CCodeDeps.append(self.PlaceMacro(str(Dep),
> > + self.Macros))
> > # Add inclusion-dependencies
> > if len(T.Inputs) == 1 and T.Inputs[0] in FileDependencyDict:
> > for F in FileDependencyDict[T.Inputs[0]]:
> > Deps.append(self.PlaceMacro(str(F), self.Macros))
> > # Add source-dependencies @@ -971,16 +991,63 @@
> > cleanlib:
> > if T.GenFileListMacro:
> > Deps.append("$(%s)" % T.FileListMacro)
> > if Type in [TAB_OBJECT_FILE, TAB_STATIC_LIBRARY]:
> > Deps.append("$(%s)" % T.ListFileMacro)
> >
> > - TargetDict = {
> > - "target" : self.PlaceMacro(T.Target.Path, self.Macros),
> > - "cmd" : "\n\t".join(T.Commands),
> > - "deps" : Deps
> > - }
> > - self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDict))
> > + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE:
> > + T, CmdTarget, CmdTargetDict, CmdCppDict =
> > + self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict,
> > CmdCppDict, DependencyDict)
> > + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps":
> > CCodeDeps}
> > + CmdLine = self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIST', '$(OBJLIST')
> > + if T.Commands:
> > + CmdLine = '%s%s' %(CmdLine, TAB_LINE_BREAK)
> > + if CCodeDeps or CmdLine:
> > + self.BuildTargetList.append(CmdLine)
> > + else:
> > + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": Deps}
> > +
> > + self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(Targ
> > + etDict))
> > +
> > + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict):
> > + if not CmdSumDict:
> > + for item in self._AutoGenObject.Targets[Type]:
> > + CmdSumDict[item.Target.SubDir] = item.Target.BaseName
> > + for CppPath in item.Inputs:
> > + Path = self.PlaceMacro(CppPath.Path, self.Macros)
> > + if CmdCppDict.get(item.Target.SubDir):
> > + CmdCppDict[item.Target.SubDir].append(Path)
> > + else:
> > + CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path]
> > + if CppPath.Path in DependencyDict:
> > + for Temp in DependencyDict[CppPath.Path]:
> > + Path = self.PlaceMacro(Temp.Path, self.Macros)
> > + if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]):
> > + CmdCppDict[item.Target.SubDir].append(Path)
> > + if T.Commands:
> > + CommandList = T.Commands[:]
> > + for Item in CommandList[:]:
> > + SingleCommandList = Item.split()
> > + if len(SingleCommandList) > 0 and '$(CC)' in SingleCommandList[0]:
> > + for Temp in SingleCommandList:
> > + if Temp.startswith('/Fo'):
> > + CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH)
> > + break
> > + else: continue
> > + if CmdSign not in list(CmdTargetDict.keys()):
> > + CmdLine = Item.replace(Temp, CmdSign)
> > + if GlobalData.gOptions.ThreadNumber:
> > + CmdLine = CmdLine.replace('/MP7', '%s%d' % ('/MP', GlobalData.gOptions.ThreadNumber))
> > + CmdTargetDict[CmdSign] = CmdLine
> > + else:
> > + CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1])
> > + Index = CommandList.index(Item)
> > + CommandList.pop(Index)
> > + if SingleCommandList[-1].endswith("%s%s.c" %
> > + (TAB_SLASH, CmdSumDict[CmdSign.lstrip('/Fo').rsplit(TAB_SLASH,
> > 1)[0]])):
> > + Cpplist = CmdCppDict[T.Target.SubDir]
> > + Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir))
> > + T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign])
> > + else:
> > + T.Commands.pop(Index)
> > + return T, CmdSumDict, CmdTargetDict, CmdCppDict
> >
> > ## For creating makefile targets for dependent libraries
> > def ProcessDependentLibrary(self):
> > for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList:
> > if not LibraryAutoGen.IsBinaryModule:
> > --
> > 2.20.1.windows.1
^ permalink raw reply [flat|nested] 8+ messages in thread
end of thread, other threads:[~2019-04-03 2:45 UTC | newest]
Thread overview: 8+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2019-03-29 12:30 [Patch] BaseTools:Enable the /MP option of MSVC compiler Feng, Bob C
2019-03-29 14:20 ` Gao, Liming
2019-04-03 2:11 ` Feng, Bob C
2019-04-03 2:45 ` Gao, Liming
2019-03-30 1:42 ` Ni, Ray
2019-03-30 8:14 ` Minnow Ware
2019-04-01 14:55 ` Feng, Bob C
2019-03-30 11:19 ` Feng, Bob C
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox