* [Patch] BaseTools:Enable the /MP option of MSVC compiler @ 2019-03-29 12:30 Feng, Bob C 2019-03-29 14:20 ` Gao, Liming 2019-03-30 1:42 ` Ni, Ray 0 siblings, 2 replies; 8+ messages in thread From: Feng, Bob C @ 2019-03-29 12:30 UTC (permalink / raw) To: edk2-devel; +Cc: Zhiju Fan, Bob Feng, Liming Gao From: Zhiju Fan <zhijux.fan@intel.com> https://bugzilla.tianocore.org/show_bug.cgi?id=1672 The /MP option of MSVC compiler can reduce the total time to compile the source files on the command line. This patch is going to enable this MSVC option in BaseTools. Cc: Bob Feng <bob.c.feng@intel.com> Cc: Liming Gao <liming.gao@intel.com> Contributed-under: TianoCore Contribution Agreement 1.1 Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com> --- BaseTools/Conf/build_rule.template | 2 +- BaseTools/Source/Python/AutoGen/GenMake.py | 81 ++++++++++++++++++++-- 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/BaseTools/Conf/build_rule.template b/BaseTools/Conf/build_rule.template index e56b1d9c59..e7d736740f 100755 --- a/BaseTools/Conf/build_rule.template +++ b/BaseTools/Conf/build_rule.template @@ -127,11 +127,11 @@ <OutputFile> $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj <Command.MSFT, Command.INTEL> - "$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src} + "$(CC)" /MP7 /Fo${d_path}\ $(CC_FLAGS) $(INC) ${src} <Command.GCC, Command.RVCT> # For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues "$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src} diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py index b441817b52..04951346ad 100644 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py @@ -433,11 +433,11 @@ cleanlib: self.BuildTargetList = [] # [target string] self.PendingBuildTargetList = [] # [FileBuildRule objects] self.CommonFileDependency = [] self.FileListMacros = {} self.ListFileMacros = {} - + self.ObjTargetDict = {} self.FileCache = {} self.LibraryBuildCommandList = [] self.LibraryFileList = [] self.LibraryMakefileList = [] self.LibraryBuildDirectoryList = [] @@ -616,10 +616,15 @@ cleanlib: ListFileName, "\n".join(self.ListFileMacros[ListFileMacro]), False ) + # Generate objlist used to create .obj file + for Type in self.ObjTargetDict: + NewLine = ' '.join(list(self.ObjTargetDict[Type])) + FileMacroList.append("OBJLIST_%s = %s" % (list(self.ObjTargetDict.keys()).index(Type), NewLine)) + BcTargetList = [] MakefileName = self._FILE_NAME_[self._FileType] LibraryMakeCommandList = [] for D in self.LibraryBuildDirectoryList: @@ -925,17 +930,22 @@ cleanlib: # Extract common files list in the dependency files # for File in DepSet: self.CommonFileDependency.append(self.PlaceMacro(File.Path, self.Macros)) + CmdSumDict = {} + CmdTargetDict = {} + CmdCppDict = {} + DependencyDict = FileDependencyDict.copy() for File in FileDependencyDict: # skip non-C files if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c": continue NewDepSet = set(FileDependencyDict[File]) NewDepSet -= DepSet FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet) + DependencyDict[File] = list(NewDepSet) # Convert target description object to target string in makefile for Type in self._AutoGenObject.Targets: for T in self._AutoGenObject.Targets[Type]: # Generate related macros if needed @@ -943,15 +953,25 @@ cleanlib: self.FileListMacros[T.FileListMacro] = [] if T.GenListFile and T.ListFileMacro not in self.ListFileMacros: self.ListFileMacros[T.ListFileMacro] = [] if T.GenIncListFile and T.IncListFileMacro not in self.ListFileMacros: self.ListFileMacros[T.IncListFileMacro] = [] + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE: + NewFile = self.PlaceMacro(str(T), self.Macros) + if self.ObjTargetDict.get(T.Target.SubDir): + self.ObjTargetDict[T.Target.SubDir].add(NewFile) + else: + self.ObjTargetDict[T.Target.SubDir] = set() + self.ObjTargetDict[T.Target.SubDir].add(NewFile) Deps = [] + CCodeDeps = [] # Add force-dependencies for Dep in T.Dependencies: Deps.append(self.PlaceMacro(str(Dep), self.Macros)) + if Dep != '$(MAKE_FILE)': + CCodeDeps.append(self.PlaceMacro(str(Dep), self.Macros)) # Add inclusion-dependencies if len(T.Inputs) == 1 and T.Inputs[0] in FileDependencyDict: for F in FileDependencyDict[T.Inputs[0]]: Deps.append(self.PlaceMacro(str(F), self.Macros)) # Add source-dependencies @@ -971,16 +991,63 @@ cleanlib: if T.GenFileListMacro: Deps.append("$(%s)" % T.FileListMacro) if Type in [TAB_OBJECT_FILE, TAB_STATIC_LIBRARY]: Deps.append("$(%s)" % T.ListFileMacro) - TargetDict = { - "target" : self.PlaceMacro(T.Target.Path, self.Macros), - "cmd" : "\n\t".join(T.Commands), - "deps" : Deps - } - self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDict)) + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE: + T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict) + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps} + CmdLine = self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIST', '$(OBJLIST') + if T.Commands: + CmdLine = '%s%s' %(CmdLine, TAB_LINE_BREAK) + if CCodeDeps or CmdLine: + self.BuildTargetList.append(CmdLine) + else: + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": Deps} + self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDict)) + + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict): + if not CmdSumDict: + for item in self._AutoGenObject.Targets[Type]: + CmdSumDict[item.Target.SubDir] = item.Target.BaseName + for CppPath in item.Inputs: + Path = self.PlaceMacro(CppPath.Path, self.Macros) + if CmdCppDict.get(item.Target.SubDir): + CmdCppDict[item.Target.SubDir].append(Path) + else: + CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path] + if CppPath.Path in DependencyDict: + for Temp in DependencyDict[CppPath.Path]: + Path = self.PlaceMacro(Temp.Path, self.Macros) + if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]): + CmdCppDict[item.Target.SubDir].append(Path) + if T.Commands: + CommandList = T.Commands[:] + for Item in CommandList[:]: + SingleCommandList = Item.split() + if len(SingleCommandList) > 0 and '$(CC)' in SingleCommandList[0]: + for Temp in SingleCommandList: + if Temp.startswith('/Fo'): + CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH) + break + else: continue + if CmdSign not in list(CmdTargetDict.keys()): + CmdLine = Item.replace(Temp, CmdSign) + if GlobalData.gOptions.ThreadNumber: + CmdLine = CmdLine.replace('/MP7', '%s%d' % ('/MP', GlobalData.gOptions.ThreadNumber)) + CmdTargetDict[CmdSign] = CmdLine + else: + CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1]) + Index = CommandList.index(Item) + CommandList.pop(Index) + if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign.lstrip('/Fo').rsplit(TAB_SLASH, 1)[0]])): + Cpplist = CmdCppDict[T.Target.SubDir] + Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir)) + T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign]) + else: + T.Commands.pop(Index) + return T, CmdSumDict, CmdTargetDict, CmdCppDict ## For creating makefile targets for dependent libraries def ProcessDependentLibrary(self): for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList: if not LibraryAutoGen.IsBinaryModule: -- 2.20.1.windows.1 ^ permalink raw reply related [flat|nested] 8+ messages in thread
* Re: [Patch] BaseTools:Enable the /MP option of MSVC compiler 2019-03-29 12:30 [Patch] BaseTools:Enable the /MP option of MSVC compiler Feng, Bob C @ 2019-03-29 14:20 ` Gao, Liming 2019-04-03 2:11 ` Feng, Bob C 2019-03-30 1:42 ` Ni, Ray 1 sibling, 1 reply; 8+ messages in thread From: Gao, Liming @ 2019-03-29 14:20 UTC (permalink / raw) To: Feng, Bob C, edk2-devel@lists.01.org Bob: How about use /MP option without process number? The compiler will retrieves the number of effective processors on your computer. And, move this option into CC_FLAGS in tools_def.txt. If so, platform can override it in platform DSC file. Last, please provide the performance data with this option. Thanks Liming > -----Original Message----- > From: Feng, Bob C > Sent: Friday, March 29, 2019 8:30 PM > To: edk2-devel@lists.01.org > Cc: Fan, ZhijuX <zhijux.fan@intel.com>; Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming <liming.gao@intel.com> > Subject: [Patch] BaseTools:Enable the /MP option of MSVC compiler > > From: Zhiju Fan <zhijux.fan@intel.com> > > https://bugzilla.tianocore.org/show_bug.cgi?id=1672 > The /MP option of MSVC compiler can reduce the total > time to compile the source files on the command line. > > This patch is going to enable this MSVC option in BaseTools. > > Cc: Bob Feng <bob.c.feng@intel.com> > Cc: Liming Gao <liming.gao@intel.com> > Contributed-under: TianoCore Contribution Agreement 1.1 > Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com> > --- > BaseTools/Conf/build_rule.template | 2 +- > BaseTools/Source/Python/AutoGen/GenMake.py | 81 ++++++++++++++++++++-- > 2 files changed, 75 insertions(+), 8 deletions(-) > > diff --git a/BaseTools/Conf/build_rule.template b/BaseTools/Conf/build_rule.template > index e56b1d9c59..e7d736740f 100755 > --- a/BaseTools/Conf/build_rule.template > +++ b/BaseTools/Conf/build_rule.template > @@ -127,11 +127,11 @@ > > <OutputFile> > $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj > > <Command.MSFT, Command.INTEL> > - "$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src} > + "$(CC)" /MP7 /Fo${d_path}\ $(CC_FLAGS) $(INC) ${src} > > <Command.GCC, Command.RVCT> > # For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues > "$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src} > > diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py > index b441817b52..04951346ad 100644 > --- a/BaseTools/Source/Python/AutoGen/GenMake.py > +++ b/BaseTools/Source/Python/AutoGen/GenMake.py > @@ -433,11 +433,11 @@ cleanlib: > self.BuildTargetList = [] # [target string] > self.PendingBuildTargetList = [] # [FileBuildRule objects] > self.CommonFileDependency = [] > self.FileListMacros = {} > self.ListFileMacros = {} > - > + self.ObjTargetDict = {} > self.FileCache = {} > self.LibraryBuildCommandList = [] > self.LibraryFileList = [] > self.LibraryMakefileList = [] > self.LibraryBuildDirectoryList = [] > @@ -616,10 +616,15 @@ cleanlib: > ListFileName, > "\n".join(self.ListFileMacros[ListFileMacro]), > False > ) > > + # Generate objlist used to create .obj file > + for Type in self.ObjTargetDict: > + NewLine = ' '.join(list(self.ObjTargetDict[Type])) > + FileMacroList.append("OBJLIST_%s = %s" % (list(self.ObjTargetDict.keys()).index(Type), NewLine)) > + > BcTargetList = [] > > MakefileName = self._FILE_NAME_[self._FileType] > LibraryMakeCommandList = [] > for D in self.LibraryBuildDirectoryList: > @@ -925,17 +930,22 @@ cleanlib: > # Extract common files list in the dependency files > # > for File in DepSet: > self.CommonFileDependency.append(self.PlaceMacro(File.Path, self.Macros)) > > + CmdSumDict = {} > + CmdTargetDict = {} > + CmdCppDict = {} > + DependencyDict = FileDependencyDict.copy() > for File in FileDependencyDict: > # skip non-C files > if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c": > continue > NewDepSet = set(FileDependencyDict[File]) > NewDepSet -= DepSet > FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet) > + DependencyDict[File] = list(NewDepSet) > > # Convert target description object to target string in makefile > for Type in self._AutoGenObject.Targets: > for T in self._AutoGenObject.Targets[Type]: > # Generate related macros if needed > @@ -943,15 +953,25 @@ cleanlib: > self.FileListMacros[T.FileListMacro] = [] > if T.GenListFile and T.ListFileMacro not in self.ListFileMacros: > self.ListFileMacros[T.ListFileMacro] = [] > if T.GenIncListFile and T.IncListFileMacro not in self.ListFileMacros: > self.ListFileMacros[T.IncListFileMacro] = [] > + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE: > + NewFile = self.PlaceMacro(str(T), self.Macros) > + if self.ObjTargetDict.get(T.Target.SubDir): > + self.ObjTargetDict[T.Target.SubDir].add(NewFile) > + else: > + self.ObjTargetDict[T.Target.SubDir] = set() > + self.ObjTargetDict[T.Target.SubDir].add(NewFile) > > Deps = [] > + CCodeDeps = [] > # Add force-dependencies > for Dep in T.Dependencies: > Deps.append(self.PlaceMacro(str(Dep), self.Macros)) > + if Dep != '$(MAKE_FILE)': > + CCodeDeps.append(self.PlaceMacro(str(Dep), self.Macros)) > # Add inclusion-dependencies > if len(T.Inputs) == 1 and T.Inputs[0] in FileDependencyDict: > for F in FileDependencyDict[T.Inputs[0]]: > Deps.append(self.PlaceMacro(str(F), self.Macros)) > # Add source-dependencies > @@ -971,16 +991,63 @@ cleanlib: > if T.GenFileListMacro: > Deps.append("$(%s)" % T.FileListMacro) > if Type in [TAB_OBJECT_FILE, TAB_STATIC_LIBRARY]: > Deps.append("$(%s)" % T.ListFileMacro) > > - TargetDict = { > - "target" : self.PlaceMacro(T.Target.Path, self.Macros), > - "cmd" : "\n\t".join(T.Commands), > - "deps" : Deps > - } > - self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDict)) > + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE: > + T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, > CmdCppDict, DependencyDict) > + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": > CCodeDeps} > + CmdLine = self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIST', '$(OBJLIST') > + if T.Commands: > + CmdLine = '%s%s' %(CmdLine, TAB_LINE_BREAK) > + if CCodeDeps or CmdLine: > + self.BuildTargetList.append(CmdLine) > + else: > + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": Deps} > + self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDict)) > + > + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict): > + if not CmdSumDict: > + for item in self._AutoGenObject.Targets[Type]: > + CmdSumDict[item.Target.SubDir] = item.Target.BaseName > + for CppPath in item.Inputs: > + Path = self.PlaceMacro(CppPath.Path, self.Macros) > + if CmdCppDict.get(item.Target.SubDir): > + CmdCppDict[item.Target.SubDir].append(Path) > + else: > + CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path] > + if CppPath.Path in DependencyDict: > + for Temp in DependencyDict[CppPath.Path]: > + Path = self.PlaceMacro(Temp.Path, self.Macros) > + if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]): > + CmdCppDict[item.Target.SubDir].append(Path) > + if T.Commands: > + CommandList = T.Commands[:] > + for Item in CommandList[:]: > + SingleCommandList = Item.split() > + if len(SingleCommandList) > 0 and '$(CC)' in SingleCommandList[0]: > + for Temp in SingleCommandList: > + if Temp.startswith('/Fo'): > + CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH) > + break > + else: continue > + if CmdSign not in list(CmdTargetDict.keys()): > + CmdLine = Item.replace(Temp, CmdSign) > + if GlobalData.gOptions.ThreadNumber: > + CmdLine = CmdLine.replace('/MP7', '%s%d' % ('/MP', GlobalData.gOptions.ThreadNumber)) > + CmdTargetDict[CmdSign] = CmdLine > + else: > + CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1]) > + Index = CommandList.index(Item) > + CommandList.pop(Index) > + if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign.lstrip('/Fo').rsplit(TAB_SLASH, > 1)[0]])): > + Cpplist = CmdCppDict[T.Target.SubDir] > + Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir)) > + T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign]) > + else: > + T.Commands.pop(Index) > + return T, CmdSumDict, CmdTargetDict, CmdCppDict > > ## For creating makefile targets for dependent libraries > def ProcessDependentLibrary(self): > for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList: > if not LibraryAutoGen.IsBinaryModule: > -- > 2.20.1.windows.1 ^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Patch] BaseTools:Enable the /MP option of MSVC compiler 2019-03-29 14:20 ` Gao, Liming @ 2019-04-03 2:11 ` Feng, Bob C 2019-04-03 2:45 ` Gao, Liming 0 siblings, 1 reply; 8+ messages in thread From: Feng, Bob C @ 2019-04-03 2:11 UTC (permalink / raw) To: Gao, Liming, edk2-devel@lists.01.org Liming, I add the performance data on the BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1672 Thanks, Bob -----Original Message----- From: Gao, Liming Sent: Friday, March 29, 2019 10:20 PM To: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org Cc: Fan, ZhijuX <zhijux.fan@intel.com> Subject: RE: [Patch] BaseTools:Enable the /MP option of MSVC compiler Bob: How about use /MP option without process number? The compiler will retrieves the number of effective processors on your computer. And, move this option into CC_FLAGS in tools_def.txt. If so, platform can override it in platform DSC file. Last, please provide the performance data with this option. Thanks Liming > -----Original Message----- > From: Feng, Bob C > Sent: Friday, March 29, 2019 8:30 PM > To: edk2-devel@lists.01.org > Cc: Fan, ZhijuX <zhijux.fan@intel.com>; Feng, Bob C > <bob.c.feng@intel.com>; Gao, Liming <liming.gao@intel.com> > Subject: [Patch] BaseTools:Enable the /MP option of MSVC compiler > > From: Zhiju Fan <zhijux.fan@intel.com> > > https://bugzilla.tianocore.org/show_bug.cgi?id=1672 > The /MP option of MSVC compiler can reduce the total time to compile > the source files on the command line. > > This patch is going to enable this MSVC option in BaseTools. > > Cc: Bob Feng <bob.c.feng@intel.com> > Cc: Liming Gao <liming.gao@intel.com> > Contributed-under: TianoCore Contribution Agreement 1.1 > Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com> > --- > BaseTools/Conf/build_rule.template | 2 +- > BaseTools/Source/Python/AutoGen/GenMake.py | 81 > ++++++++++++++++++++-- > 2 files changed, 75 insertions(+), 8 deletions(-) > > diff --git a/BaseTools/Conf/build_rule.template > b/BaseTools/Conf/build_rule.template > index e56b1d9c59..e7d736740f 100755 > --- a/BaseTools/Conf/build_rule.template > +++ b/BaseTools/Conf/build_rule.template > @@ -127,11 +127,11 @@ > > <OutputFile> > $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj > > <Command.MSFT, Command.INTEL> > - "$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src} > + "$(CC)" /MP7 /Fo${d_path}\ $(CC_FLAGS) $(INC) ${src} > > <Command.GCC, Command.RVCT> > # For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues > "$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src} > > diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py > b/BaseTools/Source/Python/AutoGen/GenMake.py > index b441817b52..04951346ad 100644 > --- a/BaseTools/Source/Python/AutoGen/GenMake.py > +++ b/BaseTools/Source/Python/AutoGen/GenMake.py > @@ -433,11 +433,11 @@ cleanlib: > self.BuildTargetList = [] # [target string] > self.PendingBuildTargetList = [] # [FileBuildRule objects] > self.CommonFileDependency = [] > self.FileListMacros = {} > self.ListFileMacros = {} > - > + self.ObjTargetDict = {} > self.FileCache = {} > self.LibraryBuildCommandList = [] > self.LibraryFileList = [] > self.LibraryMakefileList = [] > self.LibraryBuildDirectoryList = [] @@ -616,10 +616,15 @@ > cleanlib: > ListFileName, > "\n".join(self.ListFileMacros[ListFileMacro]), > False > ) > > + # Generate objlist used to create .obj file > + for Type in self.ObjTargetDict: > + NewLine = ' '.join(list(self.ObjTargetDict[Type])) > + FileMacroList.append("OBJLIST_%s = %s" % > + (list(self.ObjTargetDict.keys()).index(Type), NewLine)) > + > BcTargetList = [] > > MakefileName = self._FILE_NAME_[self._FileType] > LibraryMakeCommandList = [] > for D in self.LibraryBuildDirectoryList: > @@ -925,17 +930,22 @@ cleanlib: > # Extract common files list in the dependency files > # > for File in DepSet: > > self.CommonFileDependency.append(self.PlaceMacro(File.Path, > self.Macros)) > > + CmdSumDict = {} > + CmdTargetDict = {} > + CmdCppDict = {} > + DependencyDict = FileDependencyDict.copy() > for File in FileDependencyDict: > # skip non-C files > if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c": > continue > NewDepSet = set(FileDependencyDict[File]) > NewDepSet -= DepSet > FileDependencyDict[File] = ["$(COMMON_DEPS)"] + > list(NewDepSet) > + DependencyDict[File] = list(NewDepSet) > > # Convert target description object to target string in makefile > for Type in self._AutoGenObject.Targets: > for T in self._AutoGenObject.Targets[Type]: > # Generate related macros if needed @@ -943,15 > +953,25 @@ cleanlib: > self.FileListMacros[T.FileListMacro] = [] > if T.GenListFile and T.ListFileMacro not in self.ListFileMacros: > self.ListFileMacros[T.ListFileMacro] = [] > if T.GenIncListFile and T.IncListFileMacro not in self.ListFileMacros: > self.ListFileMacros[T.IncListFileMacro] = [] > + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE: > + NewFile = self.PlaceMacro(str(T), self.Macros) > + if self.ObjTargetDict.get(T.Target.SubDir): > + self.ObjTargetDict[T.Target.SubDir].add(NewFile) > + else: > + self.ObjTargetDict[T.Target.SubDir] = set() > + > + self.ObjTargetDict[T.Target.SubDir].add(NewFile) > > Deps = [] > + CCodeDeps = [] > # Add force-dependencies > for Dep in T.Dependencies: > Deps.append(self.PlaceMacro(str(Dep), > self.Macros)) > + if Dep != '$(MAKE_FILE)': > + CCodeDeps.append(self.PlaceMacro(str(Dep), > + self.Macros)) > # Add inclusion-dependencies > if len(T.Inputs) == 1 and T.Inputs[0] in FileDependencyDict: > for F in FileDependencyDict[T.Inputs[0]]: > Deps.append(self.PlaceMacro(str(F), self.Macros)) > # Add source-dependencies @@ -971,16 +991,63 @@ > cleanlib: > if T.GenFileListMacro: > Deps.append("$(%s)" % T.FileListMacro) > if Type in [TAB_OBJECT_FILE, TAB_STATIC_LIBRARY]: > Deps.append("$(%s)" % T.ListFileMacro) > > - TargetDict = { > - "target" : self.PlaceMacro(T.Target.Path, self.Macros), > - "cmd" : "\n\t".join(T.Commands), > - "deps" : Deps > - } > - self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDict)) > + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE: > + T, CmdTarget, CmdTargetDict, CmdCppDict = > + self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, > CmdCppDict, DependencyDict) > + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": > CCodeDeps} > + CmdLine = self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIST', '$(OBJLIST') > + if T.Commands: > + CmdLine = '%s%s' %(CmdLine, TAB_LINE_BREAK) > + if CCodeDeps or CmdLine: > + self.BuildTargetList.append(CmdLine) > + else: > + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": Deps} > + > + self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(Targ > + etDict)) > + > + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict): > + if not CmdSumDict: > + for item in self._AutoGenObject.Targets[Type]: > + CmdSumDict[item.Target.SubDir] = item.Target.BaseName > + for CppPath in item.Inputs: > + Path = self.PlaceMacro(CppPath.Path, self.Macros) > + if CmdCppDict.get(item.Target.SubDir): > + CmdCppDict[item.Target.SubDir].append(Path) > + else: > + CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path] > + if CppPath.Path in DependencyDict: > + for Temp in DependencyDict[CppPath.Path]: > + Path = self.PlaceMacro(Temp.Path, self.Macros) > + if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]): > + CmdCppDict[item.Target.SubDir].append(Path) > + if T.Commands: > + CommandList = T.Commands[:] > + for Item in CommandList[:]: > + SingleCommandList = Item.split() > + if len(SingleCommandList) > 0 and '$(CC)' in SingleCommandList[0]: > + for Temp in SingleCommandList: > + if Temp.startswith('/Fo'): > + CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH) > + break > + else: continue > + if CmdSign not in list(CmdTargetDict.keys()): > + CmdLine = Item.replace(Temp, CmdSign) > + if GlobalData.gOptions.ThreadNumber: > + CmdLine = CmdLine.replace('/MP7', '%s%d' % ('/MP', GlobalData.gOptions.ThreadNumber)) > + CmdTargetDict[CmdSign] = CmdLine > + else: > + CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1]) > + Index = CommandList.index(Item) > + CommandList.pop(Index) > + if SingleCommandList[-1].endswith("%s%s.c" % > + (TAB_SLASH, CmdSumDict[CmdSign.lstrip('/Fo').rsplit(TAB_SLASH, > 1)[0]])): > + Cpplist = CmdCppDict[T.Target.SubDir] > + Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir)) > + T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign]) > + else: > + T.Commands.pop(Index) > + return T, CmdSumDict, CmdTargetDict, CmdCppDict > > ## For creating makefile targets for dependent libraries > def ProcessDependentLibrary(self): > for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList: > if not LibraryAutoGen.IsBinaryModule: > -- > 2.20.1.windows.1 ^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Patch] BaseTools:Enable the /MP option of MSVC compiler 2019-04-03 2:11 ` Feng, Bob C @ 2019-04-03 2:45 ` Gao, Liming 0 siblings, 0 replies; 8+ messages in thread From: Gao, Liming @ 2019-04-03 2:45 UTC (permalink / raw) To: Feng, Bob C, edk2-devel@lists.01.org The clean build performance data is good. This change updates build rule generation for C source files on MSFT tool chain. It is a base tool behavior change. If it doesn't depend on build rule, could you drop the change in build rule.txt. MP flag will be moved to tools_def.txt. Thanks Liming > -----Original Message----- > From: Feng, Bob C > Sent: Wednesday, April 3, 2019 10:12 AM > To: Gao, Liming <liming.gao@intel.com>; edk2-devel@lists.01.org > Cc: Fan, ZhijuX <zhijux.fan@intel.com> > Subject: RE: [Patch] BaseTools:Enable the /MP option of MSVC compiler > > Liming, > > I add the performance data on the BZ: > https://bugzilla.tianocore.org/show_bug.cgi?id=1672 > > Thanks, > Bob > > -----Original Message----- > From: Gao, Liming > Sent: Friday, March 29, 2019 10:20 PM > To: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org > Cc: Fan, ZhijuX <zhijux.fan@intel.com> > Subject: RE: [Patch] BaseTools:Enable the /MP option of MSVC compiler > > Bob: > How about use /MP option without process number? The compiler will retrieves the number of effective processors on your computer. > And, move this option into CC_FLAGS in tools_def.txt. If so, platform can override it in platform DSC file. > > Last, please provide the performance data with this option. > > Thanks > Liming > > -----Original Message----- > > From: Feng, Bob C > > Sent: Friday, March 29, 2019 8:30 PM > > To: edk2-devel@lists.01.org > > Cc: Fan, ZhijuX <zhijux.fan@intel.com>; Feng, Bob C > > <bob.c.feng@intel.com>; Gao, Liming <liming.gao@intel.com> > > Subject: [Patch] BaseTools:Enable the /MP option of MSVC compiler > > > > From: Zhiju Fan <zhijux.fan@intel.com> > > > > https://bugzilla.tianocore.org/show_bug.cgi?id=1672 > > The /MP option of MSVC compiler can reduce the total time to compile > > the source files on the command line. > > > > This patch is going to enable this MSVC option in BaseTools. > > > > Cc: Bob Feng <bob.c.feng@intel.com> > > Cc: Liming Gao <liming.gao@intel.com> > > Contributed-under: TianoCore Contribution Agreement 1.1 > > Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com> > > --- > > BaseTools/Conf/build_rule.template | 2 +- > > BaseTools/Source/Python/AutoGen/GenMake.py | 81 > > ++++++++++++++++++++-- > > 2 files changed, 75 insertions(+), 8 deletions(-) > > > > diff --git a/BaseTools/Conf/build_rule.template > > b/BaseTools/Conf/build_rule.template > > index e56b1d9c59..e7d736740f 100755 > > --- a/BaseTools/Conf/build_rule.template > > +++ b/BaseTools/Conf/build_rule.template > > @@ -127,11 +127,11 @@ > > > > <OutputFile> > > $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj > > > > <Command.MSFT, Command.INTEL> > > - "$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src} > > + "$(CC)" /MP7 /Fo${d_path}\ $(CC_FLAGS) $(INC) ${src} > > > > <Command.GCC, Command.RVCT> > > # For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues > > "$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src} > > > > diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py > > b/BaseTools/Source/Python/AutoGen/GenMake.py > > index b441817b52..04951346ad 100644 > > --- a/BaseTools/Source/Python/AutoGen/GenMake.py > > +++ b/BaseTools/Source/Python/AutoGen/GenMake.py > > @@ -433,11 +433,11 @@ cleanlib: > > self.BuildTargetList = [] # [target string] > > self.PendingBuildTargetList = [] # [FileBuildRule objects] > > self.CommonFileDependency = [] > > self.FileListMacros = {} > > self.ListFileMacros = {} > > - > > + self.ObjTargetDict = {} > > self.FileCache = {} > > self.LibraryBuildCommandList = [] > > self.LibraryFileList = [] > > self.LibraryMakefileList = [] > > self.LibraryBuildDirectoryList = [] @@ -616,10 +616,15 @@ > > cleanlib: > > ListFileName, > > "\n".join(self.ListFileMacros[ListFileMacro]), > > False > > ) > > > > + # Generate objlist used to create .obj file > > + for Type in self.ObjTargetDict: > > + NewLine = ' '.join(list(self.ObjTargetDict[Type])) > > + FileMacroList.append("OBJLIST_%s = %s" % > > + (list(self.ObjTargetDict.keys()).index(Type), NewLine)) > > + > > BcTargetList = [] > > > > MakefileName = self._FILE_NAME_[self._FileType] > > LibraryMakeCommandList = [] > > for D in self.LibraryBuildDirectoryList: > > @@ -925,17 +930,22 @@ cleanlib: > > # Extract common files list in the dependency files > > # > > for File in DepSet: > > > > self.CommonFileDependency.append(self.PlaceMacro(File.Path, > > self.Macros)) > > > > + CmdSumDict = {} > > + CmdTargetDict = {} > > + CmdCppDict = {} > > + DependencyDict = FileDependencyDict.copy() > > for File in FileDependencyDict: > > # skip non-C files > > if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c": > > continue > > NewDepSet = set(FileDependencyDict[File]) > > NewDepSet -= DepSet > > FileDependencyDict[File] = ["$(COMMON_DEPS)"] + > > list(NewDepSet) > > + DependencyDict[File] = list(NewDepSet) > > > > # Convert target description object to target string in makefile > > for Type in self._AutoGenObject.Targets: > > for T in self._AutoGenObject.Targets[Type]: > > # Generate related macros if needed @@ -943,15 > > +953,25 @@ cleanlib: > > self.FileListMacros[T.FileListMacro] = [] > > if T.GenListFile and T.ListFileMacro not in self.ListFileMacros: > > self.ListFileMacros[T.ListFileMacro] = [] > > if T.GenIncListFile and T.IncListFileMacro not in self.ListFileMacros: > > self.ListFileMacros[T.IncListFileMacro] = [] > > + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE: > > + NewFile = self.PlaceMacro(str(T), self.Macros) > > + if self.ObjTargetDict.get(T.Target.SubDir): > > + self.ObjTargetDict[T.Target.SubDir].add(NewFile) > > + else: > > + self.ObjTargetDict[T.Target.SubDir] = set() > > + > > + self.ObjTargetDict[T.Target.SubDir].add(NewFile) > > > > Deps = [] > > + CCodeDeps = [] > > # Add force-dependencies > > for Dep in T.Dependencies: > > Deps.append(self.PlaceMacro(str(Dep), > > self.Macros)) > > + if Dep != '$(MAKE_FILE)': > > + CCodeDeps.append(self.PlaceMacro(str(Dep), > > + self.Macros)) > > # Add inclusion-dependencies > > if len(T.Inputs) == 1 and T.Inputs[0] in FileDependencyDict: > > for F in FileDependencyDict[T.Inputs[0]]: > > Deps.append(self.PlaceMacro(str(F), self.Macros)) > > # Add source-dependencies @@ -971,16 +991,63 @@ > > cleanlib: > > if T.GenFileListMacro: > > Deps.append("$(%s)" % T.FileListMacro) > > if Type in [TAB_OBJECT_FILE, TAB_STATIC_LIBRARY]: > > Deps.append("$(%s)" % T.ListFileMacro) > > > > - TargetDict = { > > - "target" : self.PlaceMacro(T.Target.Path, self.Macros), > > - "cmd" : "\n\t".join(T.Commands), > > - "deps" : Deps > > - } > > - self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDict)) > > + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE: > > + T, CmdTarget, CmdTargetDict, CmdCppDict = > > + self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, > > CmdCppDict, DependencyDict) > > + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": > > CCodeDeps} > > + CmdLine = self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIST', '$(OBJLIST') > > + if T.Commands: > > + CmdLine = '%s%s' %(CmdLine, TAB_LINE_BREAK) > > + if CCodeDeps or CmdLine: > > + self.BuildTargetList.append(CmdLine) > > + else: > > + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": Deps} > > + > > + self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(Targ > > + etDict)) > > + > > + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict): > > + if not CmdSumDict: > > + for item in self._AutoGenObject.Targets[Type]: > > + CmdSumDict[item.Target.SubDir] = item.Target.BaseName > > + for CppPath in item.Inputs: > > + Path = self.PlaceMacro(CppPath.Path, self.Macros) > > + if CmdCppDict.get(item.Target.SubDir): > > + CmdCppDict[item.Target.SubDir].append(Path) > > + else: > > + CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path] > > + if CppPath.Path in DependencyDict: > > + for Temp in DependencyDict[CppPath.Path]: > > + Path = self.PlaceMacro(Temp.Path, self.Macros) > > + if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]): > > + CmdCppDict[item.Target.SubDir].append(Path) > > + if T.Commands: > > + CommandList = T.Commands[:] > > + for Item in CommandList[:]: > > + SingleCommandList = Item.split() > > + if len(SingleCommandList) > 0 and '$(CC)' in SingleCommandList[0]: > > + for Temp in SingleCommandList: > > + if Temp.startswith('/Fo'): > > + CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH) > > + break > > + else: continue > > + if CmdSign not in list(CmdTargetDict.keys()): > > + CmdLine = Item.replace(Temp, CmdSign) > > + if GlobalData.gOptions.ThreadNumber: > > + CmdLine = CmdLine.replace('/MP7', '%s%d' % ('/MP', GlobalData.gOptions.ThreadNumber)) > > + CmdTargetDict[CmdSign] = CmdLine > > + else: > > + CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1]) > > + Index = CommandList.index(Item) > > + CommandList.pop(Index) > > + if SingleCommandList[-1].endswith("%s%s.c" % > > + (TAB_SLASH, CmdSumDict[CmdSign.lstrip('/Fo').rsplit(TAB_SLASH, > > 1)[0]])): > > + Cpplist = CmdCppDict[T.Target.SubDir] > > + Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir)) > > + T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign]) > > + else: > > + T.Commands.pop(Index) > > + return T, CmdSumDict, CmdTargetDict, CmdCppDict > > > > ## For creating makefile targets for dependent libraries > > def ProcessDependentLibrary(self): > > for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList: > > if not LibraryAutoGen.IsBinaryModule: > > -- > > 2.20.1.windows.1 ^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Patch] BaseTools:Enable the /MP option of MSVC compiler 2019-03-29 12:30 [Patch] BaseTools:Enable the /MP option of MSVC compiler Feng, Bob C 2019-03-29 14:20 ` Gao, Liming @ 2019-03-30 1:42 ` Ni, Ray 2019-03-30 8:14 ` Minnow Ware 2019-03-30 11:19 ` Feng, Bob C 1 sibling, 2 replies; 8+ messages in thread From: Ni, Ray @ 2019-03-30 1:42 UTC (permalink / raw) To: Feng, Bob C, edk2-devel@lists.01.org; +Cc: Gao, Liming https://devblogs.microsoft.com/cppblog/recommendations-to-speed-c-builds-in-visual-studio/#_PCH Bob, Per above article, It seems /Gm needs to be removed when /MP is used. > -----Original Message----- > From: edk2-devel [mailto:edk2-devel-bounces@lists.01.org] On Behalf Of Feng, > Bob C > Sent: Friday, March 29, 2019 8:30 PM > To: edk2-devel@lists.01.org > Cc: Gao, Liming <liming.gao@intel.com> > Subject: [edk2] [Patch] BaseTools:Enable the /MP option of MSVC compiler > > From: Zhiju Fan <zhijux.fan@intel.com> > > https://bugzilla.tianocore.org/show_bug.cgi?id=1672 > The /MP option of MSVC compiler can reduce the total time to compile the > source files on the command line. > > This patch is going to enable this MSVC option in BaseTools. > > Cc: Bob Feng <bob.c.feng@intel.com> > Cc: Liming Gao <liming.gao@intel.com> > Contributed-under: TianoCore Contribution Agreement 1.1 > Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com> > --- > BaseTools/Conf/build_rule.template | 2 +- > BaseTools/Source/Python/AutoGen/GenMake.py | 81 ++++++++++++++++++++- > - > 2 files changed, 75 insertions(+), 8 deletions(-) > > diff --git a/BaseTools/Conf/build_rule.template > b/BaseTools/Conf/build_rule.template > index e56b1d9c59..e7d736740f 100755 > --- a/BaseTools/Conf/build_rule.template > +++ b/BaseTools/Conf/build_rule.template > @@ -127,11 +127,11 @@ > > <OutputFile> > $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj > > <Command.MSFT, Command.INTEL> > - "$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src} > + "$(CC)" /MP7 /Fo${d_path}\ $(CC_FLAGS) $(INC) ${src} > > <Command.GCC, Command.RVCT> > # For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues > "$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src} > > diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py > b/BaseTools/Source/Python/AutoGen/GenMake.py > index b441817b52..04951346ad 100644 > --- a/BaseTools/Source/Python/AutoGen/GenMake.py > +++ b/BaseTools/Source/Python/AutoGen/GenMake.py > @@ -433,11 +433,11 @@ cleanlib: > self.BuildTargetList = [] # [target string] > self.PendingBuildTargetList = [] # [FileBuildRule objects] > self.CommonFileDependency = [] > self.FileListMacros = {} > self.ListFileMacros = {} > - > + self.ObjTargetDict = {} > self.FileCache = {} > self.LibraryBuildCommandList = [] > self.LibraryFileList = [] > self.LibraryMakefileList = [] > self.LibraryBuildDirectoryList = [] @@ -616,10 +616,15 @@ cleanlib: > ListFileName, > "\n".join(self.ListFileMacros[ListFileMacro]), > False > ) > > + # Generate objlist used to create .obj file > + for Type in self.ObjTargetDict: > + NewLine = ' '.join(list(self.ObjTargetDict[Type])) > + FileMacroList.append("OBJLIST_%s = %s" % > + (list(self.ObjTargetDict.keys()).index(Type), NewLine)) > + > BcTargetList = [] > > MakefileName = self._FILE_NAME_[self._FileType] > LibraryMakeCommandList = [] > for D in self.LibraryBuildDirectoryList: > @@ -925,17 +930,22 @@ cleanlib: > # Extract common files list in the dependency files > # > for File in DepSet: > self.CommonFileDependency.append(self.PlaceMacro(File.Path, > self.Macros)) > > + CmdSumDict = {} > + CmdTargetDict = {} > + CmdCppDict = {} > + DependencyDict = FileDependencyDict.copy() > for File in FileDependencyDict: > # skip non-C files > if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c": > continue > NewDepSet = set(FileDependencyDict[File]) > NewDepSet -= DepSet > FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet) > + DependencyDict[File] = list(NewDepSet) > > # Convert target description object to target string in makefile > for Type in self._AutoGenObject.Targets: > for T in self._AutoGenObject.Targets[Type]: > # Generate related macros if needed @@ -943,15 +953,25 @@ > cleanlib: > self.FileListMacros[T.FileListMacro] = [] > if T.GenListFile and T.ListFileMacro not in self.ListFileMacros: > self.ListFileMacros[T.ListFileMacro] = [] > if T.GenIncListFile and T.IncListFileMacro not in self.ListFileMacros: > self.ListFileMacros[T.IncListFileMacro] = [] > + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and > Type == TAB_C_CODE_FILE: > + NewFile = self.PlaceMacro(str(T), self.Macros) > + if self.ObjTargetDict.get(T.Target.SubDir): > + self.ObjTargetDict[T.Target.SubDir].add(NewFile) > + else: > + self.ObjTargetDict[T.Target.SubDir] = set() > + > + self.ObjTargetDict[T.Target.SubDir].add(NewFile) > > Deps = [] > + CCodeDeps = [] > # Add force-dependencies > for Dep in T.Dependencies: > Deps.append(self.PlaceMacro(str(Dep), self.Macros)) > + if Dep != '$(MAKE_FILE)': > + CCodeDeps.append(self.PlaceMacro(str(Dep), > + self.Macros)) > # Add inclusion-dependencies > if len(T.Inputs) == 1 and T.Inputs[0] in FileDependencyDict: > for F in FileDependencyDict[T.Inputs[0]]: > Deps.append(self.PlaceMacro(str(F), self.Macros)) > # Add source-dependencies @@ -971,16 +991,63 @@ cleanlib: > if T.GenFileListMacro: > Deps.append("$(%s)" % T.FileListMacro) > if Type in [TAB_OBJECT_FILE, TAB_STATIC_LIBRARY]: > Deps.append("$(%s)" % T.ListFileMacro) > > - TargetDict = { > - "target" : self.PlaceMacro(T.Target.Path, self.Macros), > - "cmd" : "\n\t".join(T.Commands), > - "deps" : Deps > - } > - > self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDic > t)) > + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and > Type == TAB_C_CODE_FILE: > + T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, > Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict) > + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), > "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps} > + CmdLine = > self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIS > T', '$(OBJLIST') > + if T.Commands: > + CmdLine = '%s%s' %(CmdLine, TAB_LINE_BREAK) > + if CCodeDeps or CmdLine: > + self.BuildTargetList.append(CmdLine) > + else: > + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), > "cmd": "\n\t".join(T.Commands),"deps": Deps} > + > + self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(Target > + Dict)) > + > + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, > DependencyDict): > + if not CmdSumDict: > + for item in self._AutoGenObject.Targets[Type]: > + CmdSumDict[item.Target.SubDir] = item.Target.BaseName > + for CppPath in item.Inputs: > + Path = self.PlaceMacro(CppPath.Path, self.Macros) > + if CmdCppDict.get(item.Target.SubDir): > + CmdCppDict[item.Target.SubDir].append(Path) > + else: > + CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path] > + if CppPath.Path in DependencyDict: > + for Temp in DependencyDict[CppPath.Path]: > + Path = self.PlaceMacro(Temp.Path, self.Macros) > + if Path not in (self.CommonFileDependency + > CmdCppDict[item.Target.SubDir]): > + CmdCppDict[item.Target.SubDir].append(Path) > + if T.Commands: > + CommandList = T.Commands[:] > + for Item in CommandList[:]: > + SingleCommandList = Item.split() > + if len(SingleCommandList) > 0 and '$(CC)' in SingleCommandList[0]: > + for Temp in SingleCommandList: > + if Temp.startswith('/Fo'): > + CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH) > + break > + else: continue > + if CmdSign not in list(CmdTargetDict.keys()): > + CmdLine = Item.replace(Temp, CmdSign) > + if GlobalData.gOptions.ThreadNumber: > + CmdLine = CmdLine.replace('/MP7', '%s%d' % ('/MP', > GlobalData.gOptions.ThreadNumber)) > + CmdTargetDict[CmdSign] = CmdLine > + else: > + CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], > SingleCommandList[-1]) > + Index = CommandList.index(Item) > + CommandList.pop(Index) > + if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, > CmdSumDict[CmdSign.lstrip('/Fo').rsplit(TAB_SLASH, 1)[0]])): > + Cpplist = CmdCppDict[T.Target.SubDir] > + Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' % > list(self.ObjTargetDict.keys()).index(T.Target.SubDir)) > + T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), > CmdTargetDict[CmdSign]) > + else: > + T.Commands.pop(Index) > + return T, CmdSumDict, CmdTargetDict, CmdCppDict > > ## For creating makefile targets for dependent libraries > def ProcessDependentLibrary(self): > for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList: > if not LibraryAutoGen.IsBinaryModule: > -- > 2.20.1.windows.1 > > _______________________________________________ > edk2-devel mailing list > edk2-devel@lists.01.org > https://lists.01.org/mailman/listinfo/edk2-devel ^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Patch] BaseTools:Enable the /MP option of MSVC compiler 2019-03-30 1:42 ` Ni, Ray @ 2019-03-30 8:14 ` Minnow Ware 2019-04-01 14:55 ` Feng, Bob C 2019-03-30 11:19 ` Feng, Bob C 1 sibling, 1 reply; 8+ messages in thread From: Minnow Ware @ 2019-03-30 8:14 UTC (permalink / raw) To: Ni, Ray, Feng, Bob C, edk2-devel@lists.01.org; +Cc: Gao, Liming It would also be good, to add a REBUILD target to the makefiles, that forces the compiler to translate all C-files from a component at once, w/o dependency check e.g.: REBUILD: cl /c [additional switches] file1.c file2.c file3.c file4.c so that the compiler itself is invoked only once per component. This is possible for Microsoft, Intel and GNU C-compiler. Best regards, Kilian https://github.com/MinnowWare ________________________________ From: edk2-devel <edk2-devel-bounces@lists.01.org> on behalf of Ni, Ray <ray.ni@intel.com> Sent: Saturday, March 30, 2019 2:42:21 AM To: Feng, Bob C; edk2-devel@lists.01.org Cc: Gao, Liming Subject: Re: [edk2] [Patch] BaseTools:Enable the /MP option of MSVC compiler https://devblogs.microsoft.com/cppblog/recommendations-to-speed-c-builds-in-visual-studio/#_PCH Bob, Per above article, It seems /Gm needs to be removed when /MP is used. > -----Original Message----- > From: edk2-devel [mailto:edk2-devel-bounces@lists.01.org] On Behalf Of Feng, > Bob C > Sent: Friday, March 29, 2019 8:30 PM > To: edk2-devel@lists.01.org > Cc: Gao, Liming <liming.gao@intel.com> > Subject: [edk2] [Patch] BaseTools:Enable the /MP option of MSVC compiler > > From: Zhiju Fan <zhijux.fan@intel.com> > > https://bugzilla.tianocore.org/show_bug.cgi?id=1672 > The /MP option of MSVC compiler can reduce the total time to compile the > source files on the command line. > > This patch is going to enable this MSVC option in BaseTools. > > Cc: Bob Feng <bob.c.feng@intel.com> > Cc: Liming Gao <liming.gao@intel.com> > Contributed-under: TianoCore Contribution Agreement 1.1 > Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com> > --- > BaseTools/Conf/build_rule.template | 2 +- > BaseTools/Source/Python/AutoGen/GenMake.py | 81 ++++++++++++++++++++- > - > 2 files changed, 75 insertions(+), 8 deletions(-) > > diff --git a/BaseTools/Conf/build_rule.template > b/BaseTools/Conf/build_rule.template > index e56b1d9c59..e7d736740f 100755 > --- a/BaseTools/Conf/build_rule.template > +++ b/BaseTools/Conf/build_rule.template > @@ -127,11 +127,11 @@ > > <OutputFile> > $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj > > <Command.MSFT, Command.INTEL> > - "$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src} > + "$(CC)" /MP7 /Fo${d_path}\ $(CC_FLAGS) $(INC) ${src} > > <Command.GCC, Command.RVCT> > # For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues > "$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src} > > diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py > b/BaseTools/Source/Python/AutoGen/GenMake.py > index b441817b52..04951346ad 100644 > --- a/BaseTools/Source/Python/AutoGen/GenMake.py > +++ b/BaseTools/Source/Python/AutoGen/GenMake.py > @@ -433,11 +433,11 @@ cleanlib: > self.BuildTargetList = [] # [target string] > self.PendingBuildTargetList = [] # [FileBuildRule objects] > self.CommonFileDependency = [] > self.FileListMacros = {} > self.ListFileMacros = {} > - > + self.ObjTargetDict = {} > self.FileCache = {} > self.LibraryBuildCommandList = [] > self.LibraryFileList = [] > self.LibraryMakefileList = [] > self.LibraryBuildDirectoryList = [] @@ -616,10 +616,15 @@ cleanlib: > ListFileName, > "\n".join(self.ListFileMacros[ListFileMacro]), > False > ) > > + # Generate objlist used to create .obj file > + for Type in self.ObjTargetDict: > + NewLine = ' '.join(list(self.ObjTargetDict[Type])) > + FileMacroList.append("OBJLIST_%s = %s" % > + (list(self.ObjTargetDict.keys()).index(Type), NewLine)) > + > BcTargetList = [] > > MakefileName = self._FILE_NAME_[self._FileType] > LibraryMakeCommandList = [] > for D in self.LibraryBuildDirectoryList: > @@ -925,17 +930,22 @@ cleanlib: > # Extract common files list in the dependency files > # > for File in DepSet: > self.CommonFileDependency.append(self.PlaceMacro(File.Path, > self.Macros)) > > + CmdSumDict = {} > + CmdTargetDict = {} > + CmdCppDict = {} > + DependencyDict = FileDependencyDict.copy() > for File in FileDependencyDict: > # skip non-C files > if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c": > continue > NewDepSet = set(FileDependencyDict[File]) > NewDepSet -= DepSet > FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet) > + DependencyDict[File] = list(NewDepSet) > > # Convert target description object to target string in makefile > for Type in self._AutoGenObject.Targets: > for T in self._AutoGenObject.Targets[Type]: > # Generate related macros if needed @@ -943,15 +953,25 @@ > cleanlib: > self.FileListMacros[T.FileListMacro] = [] > if T.GenListFile and T.ListFileMacro not in self.ListFileMacros: > self.ListFileMacros[T.ListFileMacro] = [] > if T.GenIncListFile and T.IncListFileMacro not in self.ListFileMacros: > self.ListFileMacros[T.IncListFileMacro] = [] > + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and > Type == TAB_C_CODE_FILE: > + NewFile = self.PlaceMacro(str(T), self.Macros) > + if self.ObjTargetDict.get(T.Target.SubDir): > + self.ObjTargetDict[T.Target.SubDir].add(NewFile) > + else: > + self.ObjTargetDict[T.Target.SubDir] = set() > + > + self.ObjTargetDict[T.Target.SubDir].add(NewFile) > > Deps = [] > + CCodeDeps = [] > # Add force-dependencies > for Dep in T.Dependencies: > Deps.append(self.PlaceMacro(str(Dep), self.Macros)) > + if Dep != '$(MAKE_FILE)': > + CCodeDeps.append(self.PlaceMacro(str(Dep), > + self.Macros)) > # Add inclusion-dependencies > if len(T.Inputs) == 1 and T.Inputs[0] in FileDependencyDict: > for F in FileDependencyDict[T.Inputs[0]]: > Deps.append(self.PlaceMacro(str(F), self.Macros)) > # Add source-dependencies @@ -971,16 +991,63 @@ cleanlib: > if T.GenFileListMacro: > Deps.append("$(%s)" % T.FileListMacro) > if Type in [TAB_OBJECT_FILE, TAB_STATIC_LIBRARY]: > Deps.append("$(%s)" % T.ListFileMacro) > > - TargetDict = { > - "target" : self.PlaceMacro(T.Target.Path, self.Macros), > - "cmd" : "\n\t".join(T.Commands), > - "deps" : Deps > - } > - > self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDic > t)) > + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and > Type == TAB_C_CODE_FILE: > + T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, > Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict) > + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), > "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps} > + CmdLine = > self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIS > T', '$(OBJLIST') > + if T.Commands: > + CmdLine = '%s%s' %(CmdLine, TAB_LINE_BREAK) > + if CCodeDeps or CmdLine: > + self.BuildTargetList.append(CmdLine) > + else: > + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), > "cmd": "\n\t".join(T.Commands),"deps": Deps} > + > + self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(Target > + Dict)) > + > + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, > DependencyDict): > + if not CmdSumDict: > + for item in self._AutoGenObject.Targets[Type]: > + CmdSumDict[item.Target.SubDir] = item.Target.BaseName > + for CppPath in item.Inputs: > + Path = self.PlaceMacro(CppPath.Path, self.Macros) > + if CmdCppDict.get(item.Target.SubDir): > + CmdCppDict[item.Target.SubDir].append(Path) > + else: > + CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path] > + if CppPath.Path in DependencyDict: > + for Temp in DependencyDict[CppPath.Path]: > + Path = self.PlaceMacro(Temp.Path, self.Macros) > + if Path not in (self.CommonFileDependency + > CmdCppDict[item.Target.SubDir]): > + CmdCppDict[item.Target.SubDir].append(Path) > + if T.Commands: > + CommandList = T.Commands[:] > + for Item in CommandList[:]: > + SingleCommandList = Item.split() > + if len(SingleCommandList) > 0 and '$(CC)' in SingleCommandList[0]: > + for Temp in SingleCommandList: > + if Temp.startswith('/Fo'): > + CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH) > + break > + else: continue > + if CmdSign not in list(CmdTargetDict.keys()): > + CmdLine = Item.replace(Temp, CmdSign) > + if GlobalData.gOptions.ThreadNumber: > + CmdLine = CmdLine.replace('/MP7', '%s%d' % ('/MP', > GlobalData.gOptions.ThreadNumber)) > + CmdTargetDict[CmdSign] = CmdLine > + else: > + CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], > SingleCommandList[-1]) > + Index = CommandList.index(Item) > + CommandList.pop(Index) > + if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, > CmdSumDict[CmdSign.lstrip('/Fo').rsplit(TAB_SLASH, 1)[0]])): > + Cpplist = CmdCppDict[T.Target.SubDir] > + Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' % > list(self.ObjTargetDict.keys()).index(T.Target.SubDir)) > + T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), > CmdTargetDict[CmdSign]) > + else: > + T.Commands.pop(Index) > + return T, CmdSumDict, CmdTargetDict, CmdCppDict > > ## For creating makefile targets for dependent libraries > def ProcessDependentLibrary(self): > for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList: > if not LibraryAutoGen.IsBinaryModule: > -- > 2.20.1.windows.1 > > _______________________________________________ > edk2-devel mailing list > edk2-devel@lists.01.org > https://lists.01.org/mailman/listinfo/edk2-devel _______________________________________________ edk2-devel mailing list edk2-devel@lists.01.org https://lists.01.org/mailman/listinfo/edk2-devel ^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Patch] BaseTools:Enable the /MP option of MSVC compiler 2019-03-30 8:14 ` Minnow Ware @ 2019-04-01 14:55 ` Feng, Bob C 0 siblings, 0 replies; 8+ messages in thread From: Feng, Bob C @ 2019-04-01 14:55 UTC (permalink / raw) To: Minnow Ware, Ni, Ray, edk2-devel@lists.01.org; +Cc: Gao, Liming Hi Kilian, It's a good suggestion. Would you please enter a new BZ for this request? I think implement this request in a separate patch would be better. Thanks, Bob From: Minnow Ware [mailto:minnowware@outlook.com] Sent: Saturday, March 30, 2019 4:15 PM To: Ni, Ray <ray.ni@intel.com>; Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org Cc: Gao, Liming <liming.gao@intel.com> Subject: RE: [edk2] [Patch] BaseTools:Enable the /MP option of MSVC compiler It would also be good, to add a REBUILD target to the makefiles, that forces the compiler to translate all C-files from a component at once, w/o dependency check e.g.: REBUILD: cl /c [additional switches] file1.c file2.c file3.c file4.c so that the compiler itself is invoked only once per component. This is possible for Microsoft, Intel and GNU C-compiler. Best regards, Kilian https://github.com/MinnowWare ________________________________ From: edk2-devel <edk2-devel-bounces@lists.01.org<mailto:edk2-devel-bounces@lists.01.org>> on behalf of Ni, Ray <ray.ni@intel.com<mailto:ray.ni@intel.com>> Sent: Saturday, March 30, 2019 2:42:21 AM To: Feng, Bob C; edk2-devel@lists.01.org<mailto:edk2-devel@lists.01.org> Cc: Gao, Liming Subject: Re: [edk2] [Patch] BaseTools:Enable the /MP option of MSVC compiler https://devblogs.microsoft.com/cppblog/recommendations-to-speed-c-builds-in-visual-studio/#_PCH Bob, Per above article, It seems /Gm needs to be removed when /MP is used. > -----Original Message----- > From: edk2-devel [mailto:edk2-devel-bounces@lists.01.org] On Behalf Of Feng, > Bob C > Sent: Friday, March 29, 2019 8:30 PM > To: edk2-devel@lists.01.org<mailto:edk2-devel@lists.01.org> > Cc: Gao, Liming <liming.gao@intel.com<mailto:liming.gao@intel.com>> > Subject: [edk2] [Patch] BaseTools:Enable the /MP option of MSVC compiler > > From: Zhiju Fan <zhijux.fan@intel.com<mailto:zhijux.fan@intel.com>> > > https://bugzilla.tianocore.org/show_bug.cgi?id=1672 > The /MP option of MSVC compiler can reduce the total time to compile the > source files on the command line. > > This patch is going to enable this MSVC option in BaseTools. > > Cc: Bob Feng <bob.c.feng@intel.com<mailto:bob.c.feng@intel.com>> > Cc: Liming Gao <liming.gao@intel.com<mailto:liming.gao@intel.com>> > Contributed-under: TianoCore Contribution Agreement 1.1 > Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com<mailto:zhijux.fan@intel.com>> > --- > BaseTools/Conf/build_rule.template | 2 +- > BaseTools/Source/Python/AutoGen/GenMake.py | 81 ++++++++++++++++++++- > - > 2 files changed, 75 insertions(+), 8 deletions(-) > > diff --git a/BaseTools/Conf/build_rule.template > b/BaseTools/Conf/build_rule.template > index e56b1d9c59..e7d736740f 100755 > --- a/BaseTools/Conf/build_rule.template > +++ b/BaseTools/Conf/build_rule.template > @@ -127,11 +127,11 @@ > > <OutputFile> > $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj > > <Command.MSFT, Command.INTEL> > - "$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src} > + "$(CC)" /MP7 /Fo${d_path}\ $(CC_FLAGS) $(INC) ${src} > > <Command.GCC, Command.RVCT> > # For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues > "$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src} > > diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py > b/BaseTools/Source/Python/AutoGen/GenMake.py > index b441817b52..04951346ad 100644 > --- a/BaseTools/Source/Python/AutoGen/GenMake.py > +++ b/BaseTools/Source/Python/AutoGen/GenMake.py > @@ -433,11 +433,11 @@ cleanlib: > self.BuildTargetList = [] # [target string] > self.PendingBuildTargetList = [] # [FileBuildRule objects] > self.CommonFileDependency = [] > self.FileListMacros = {} > self.ListFileMacros = {} > - > + self.ObjTargetDict = {} > self.FileCache = {} > self.LibraryBuildCommandList = [] > self.LibraryFileList = [] > self.LibraryMakefileList = [] > self.LibraryBuildDirectoryList = [] @@ -616,10 +616,15 @@ cleanlib: > ListFileName, > "\n".join(self.ListFileMacros[ListFileMacro]), > False > ) > > + # Generate objlist used to create .obj file > + for Type in self.ObjTargetDict: > + NewLine = ' '.join(list(self.ObjTargetDict[Type])) > + FileMacroList.append("OBJLIST_%s = %s" % > + (list(self.ObjTargetDict.keys()).index(Type), NewLine)) > + > BcTargetList = [] > > MakefileName = self._FILE_NAME_[self._FileType] > LibraryMakeCommandList = [] > for D in self.LibraryBuildDirectoryList: > @@ -925,17 +930,22 @@ cleanlib: > # Extract common files list in the dependency files > # > for File in DepSet: > self.CommonFileDependency.append(self.PlaceMacro(File.Path, > self.Macros)) > > + CmdSumDict = {} > + CmdTargetDict = {} > + CmdCppDict = {} > + DependencyDict = FileDependencyDict.copy() > for File in FileDependencyDict: > # skip non-C files > if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c": > continue > NewDepSet = set(FileDependencyDict[File]) > NewDepSet -= DepSet > FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet) > + DependencyDict[File] = list(NewDepSet) > > # Convert target description object to target string in makefile > for Type in self._AutoGenObject.Targets: > for T in self._AutoGenObject.Targets[Type]: > # Generate related macros if needed @@ -943,15 +953,25 @@ > cleanlib: > self.FileListMacros[T.FileListMacro] = [] > if T.GenListFile and T.ListFileMacro not in self.ListFileMacros: > self.ListFileMacros[T.ListFileMacro] = [] > if T.GenIncListFile and T.IncListFileMacro not in self.ListFileMacros: > self.ListFileMacros[T.IncListFileMacro] = [] > + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and > Type == TAB_C_CODE_FILE: > + NewFile = self.PlaceMacro(str(T), self.Macros) > + if self.ObjTargetDict.get(T.Target.SubDir): > + self.ObjTargetDict[T.Target.SubDir].add(NewFile) > + else: > + self.ObjTargetDict[T.Target.SubDir] = set() > + > + self.ObjTargetDict[T.Target.SubDir].add(NewFile) > > Deps = [] > + CCodeDeps = [] > # Add force-dependencies > for Dep in T.Dependencies: > Deps.append(self.PlaceMacro(str(Dep), self.Macros)) > + if Dep != '$(MAKE_FILE)': > + CCodeDeps.append(self.PlaceMacro(str(Dep), > + self.Macros)) > # Add inclusion-dependencies > if len(T.Inputs) == 1 and T.Inputs[0] in FileDependencyDict: > for F in FileDependencyDict[T.Inputs[0]]: > Deps.append(self.PlaceMacro(str(F), self.Macros)) > # Add source-dependencies @@ -971,16 +991,63 @@ cleanlib: > if T.GenFileListMacro: > Deps.append("$(%s)" % T.FileListMacro) > if Type in [TAB_OBJECT_FILE, TAB_STATIC_LIBRARY]: > Deps.append("$(%s)" % T.ListFileMacro) > > - TargetDict = { > - "target" : self.PlaceMacro(T.Target.Path, self.Macros), > - "cmd" : "\n\t".join(T.Commands), > - "deps" : Deps > - } > - > self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(TargetDic > t)) > + if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and > Type == TAB_C_CODE_FILE: > + T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, > Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict) > + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), > "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps} > + CmdLine = > self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIS > T', '$(OBJLIST') > + if T.Commands: > + CmdLine = '%s%s' %(CmdLine, TAB_LINE_BREAK) > + if CCodeDeps or CmdLine: > + self.BuildTargetList.append(CmdLine) > + else: > + TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), > "cmd": "\n\t".join(T.Commands),"deps": Deps} > + > + self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(Target > + Dict)) > + > + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, > DependencyDict): > + if not CmdSumDict: > + for item in self._AutoGenObject.Targets[Type]: > + CmdSumDict[item.Target.SubDir] = item.Target.BaseName > + for CppPath in item.Inputs: > + Path = self.PlaceMacro(CppPath.Path, self.Macros) > + if CmdCppDict.get(item.Target.SubDir): > + CmdCppDict[item.Target.SubDir].append(Path) > + else: > + CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path] > + if CppPath.Path in DependencyDict: > + for Temp in DependencyDict[CppPath.Path]: > + Path = self.PlaceMacro(Temp.Path, self.Macros) > + if Path not in (self.CommonFileDependency + > CmdCppDict[item.Target.SubDir]): > + CmdCppDict[item.Target.SubDir].append(Path) > + if T.Commands: > + CommandList = T.Commands[:] > + for Item in CommandList[:]: > + SingleCommandList = Item.split() > + if len(SingleCommandList) > 0 and '$(CC)' in SingleCommandList[0]: > + for Temp in SingleCommandList: > + if Temp.startswith('/Fo'): > + CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH) > + break > + else: continue > + if CmdSign not in list(CmdTargetDict.keys()): > + CmdLine = Item.replace(Temp, CmdSign) > + if GlobalData.gOptions.ThreadNumber: > + CmdLine = CmdLine.replace('/MP7', '%s%d' % ('/MP', > GlobalData.gOptions.ThreadNumber)) > + CmdTargetDict[CmdSign] = CmdLine > + else: > + CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], > SingleCommandList[-1]) > + Index = CommandList.index(Item) > + CommandList.pop(Index) > + if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, > CmdSumDict[CmdSign.lstrip('/Fo').rsplit(TAB_SLASH, 1)[0]])): > + Cpplist = CmdCppDict[T.Target.SubDir] > + Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' % > list(self.ObjTargetDict.keys()).index(T.Target.SubDir)) > + T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist)<file:///\\\n\t'.join(Cpplist)>, > CmdTargetDict[CmdSign]) > + else: > + T.Commands.pop(Index) > + return T, CmdSumDict, CmdTargetDict, CmdCppDict > > ## For creating makefile targets for dependent libraries > def ProcessDependentLibrary(self): > for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList: > if not LibraryAutoGen.IsBinaryModule: > -- > 2.20.1.windows.1 > > _______________________________________________ > edk2-devel mailing list > edk2-devel@lists.01.org<mailto:edk2-devel@lists.01.org> > https://lists.01.org/mailman/listinfo/edk2-devel _______________________________________________ edk2-devel mailing list edk2-devel@lists.01.org<mailto:edk2-devel@lists.01.org> https://lists.01.org/mailman/listinfo/edk2-devel ^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Patch] BaseTools:Enable the /MP option of MSVC compiler 2019-03-30 1:42 ` Ni, Ray 2019-03-30 8:14 ` Minnow Ware @ 2019-03-30 11:19 ` Feng, Bob C 1 sibling, 0 replies; 8+ messages in thread From: Feng, Bob C @ 2019-03-30 11:19 UTC (permalink / raw) To: Ni, Ray, edk2-devel@lists.01.org; +Cc: Gao, Liming Yes. Commit fb94f83131f032cd5ce027ea706c45513c1a799e removed the /Gm option from MSVC tool chain. -----Original Message----- From: Ni, Ray Sent: Saturday, March 30, 2019 9:42 AM To: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org Cc: Gao, Liming <liming.gao@intel.com> Subject: RE: [edk2] [Patch] BaseTools:Enable the /MP option of MSVC compiler https://devblogs.microsoft.com/cppblog/recommendations-to-speed-c-builds-in-visual-studio/#_PCH Bob, Per above article, It seems /Gm needs to be removed when /MP is used. > -----Original Message----- > From: edk2-devel [mailto:edk2-devel-bounces@lists.01.org] On Behalf Of > Feng, Bob C > Sent: Friday, March 29, 2019 8:30 PM > To: edk2-devel@lists.01.org > Cc: Gao, Liming <liming.gao@intel.com> > Subject: [edk2] [Patch] BaseTools:Enable the /MP option of MSVC > compiler > > From: Zhiju Fan <zhijux.fan@intel.com> > > https://bugzilla.tianocore.org/show_bug.cgi?id=1672 > The /MP option of MSVC compiler can reduce the total time to compile > the source files on the command line. > > This patch is going to enable this MSVC option in BaseTools. > > Cc: Bob Feng <bob.c.feng@intel.com> > Cc: Liming Gao <liming.gao@intel.com> > Contributed-under: TianoCore Contribution Agreement 1.1 > Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com> > --- > BaseTools/Conf/build_rule.template | 2 +- > BaseTools/Source/Python/AutoGen/GenMake.py | 81 ++++++++++++++++++++- > - > 2 files changed, 75 insertions(+), 8 deletions(-) > > diff --git a/BaseTools/Conf/build_rule.template > b/BaseTools/Conf/build_rule.template > index e56b1d9c59..e7d736740f 100755 > --- a/BaseTools/Conf/build_rule.template > +++ b/BaseTools/Conf/build_rule.template > @@ -127,11 +127,11 @@ > > <OutputFile> > $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj > > <Command.MSFT, Command.INTEL> > - "$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src} > + "$(CC)" /MP7 /Fo${d_path}\ $(CC_FLAGS) $(INC) ${src} > > <Command.GCC, Command.RVCT> > # For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues > "$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src} > > diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py > b/BaseTools/Source/Python/AutoGen/GenMake.py > index b441817b52..04951346ad 100644 > --- a/BaseTools/Source/Python/AutoGen/GenMake.py > +++ b/BaseTools/Source/Python/AutoGen/GenMake.py > @@ -433,11 +433,11 @@ cleanlib: > self.BuildTargetList = [] # [target string] > self.PendingBuildTargetList = [] # [FileBuildRule objects] > self.CommonFileDependency = [] > self.FileListMacros = {} > self.ListFileMacros = {} > - > + self.ObjTargetDict = {} > self.FileCache = {} > self.LibraryBuildCommandList = [] > self.LibraryFileList = [] > self.LibraryMakefileList = [] > self.LibraryBuildDirectoryList = [] @@ -616,10 +616,15 @@ cleanlib: > ListFileName, > "\n".join(self.ListFileMacros[ListFileMacro]), > False > ) > > + # Generate objlist used to create .obj file > + for Type in self.ObjTargetDict: > + NewLine = ' '.join(list(self.ObjTargetDict[Type])) > + FileMacroList.append("OBJLIST_%s = %s" % > + (list(self.ObjTargetDict.keys()).index(Type), NewLine)) > + > BcTargetList = [] > > MakefileName = self._FILE_NAME_[self._FileType] > LibraryMakeCommandList = [] > for D in self.LibraryBuildDirectoryList: > @@ -925,17 +930,22 @@ cleanlib: > # Extract common files list in the dependency files > # > for File in DepSet: > > self.CommonFileDependency.append(self.PlaceMacro(File.Path, > self.Macros)) > > + CmdSumDict = {} > + CmdTargetDict = {} > + CmdCppDict = {} > + DependencyDict = FileDependencyDict.copy() > for File in FileDependencyDict: > # skip non-C files > if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c": > continue > NewDepSet = set(FileDependencyDict[File]) > NewDepSet -= DepSet > FileDependencyDict[File] = ["$(COMMON_DEPS)"] + > list(NewDepSet) > + DependencyDict[File] = list(NewDepSet) > > # Convert target description object to target string in makefile > for Type in self._AutoGenObject.Targets: > for T in self._AutoGenObject.Targets[Type]: > # Generate related macros if needed @@ -943,15 > +953,25 @@ > cleanlib: > self.FileListMacros[T.FileListMacro] = [] > if T.GenListFile and T.ListFileMacro not in self.ListFileMacros: > self.ListFileMacros[T.ListFileMacro] = [] > if T.GenIncListFile and T.IncListFileMacro not in self.ListFileMacros: > self.ListFileMacros[T.IncListFileMacro] = [] > + if self._AutoGenObject.BuildRuleFamily == > + TAB_COMPILER_MSFT and > Type == TAB_C_CODE_FILE: > + NewFile = self.PlaceMacro(str(T), self.Macros) > + if self.ObjTargetDict.get(T.Target.SubDir): > + self.ObjTargetDict[T.Target.SubDir].add(NewFile) > + else: > + self.ObjTargetDict[T.Target.SubDir] = set() > + > + self.ObjTargetDict[T.Target.SubDir].add(NewFile) > > Deps = [] > + CCodeDeps = [] > # Add force-dependencies > for Dep in T.Dependencies: > Deps.append(self.PlaceMacro(str(Dep), > self.Macros)) > + if Dep != '$(MAKE_FILE)': > + CCodeDeps.append(self.PlaceMacro(str(Dep), > + self.Macros)) > # Add inclusion-dependencies > if len(T.Inputs) == 1 and T.Inputs[0] in FileDependencyDict: > for F in FileDependencyDict[T.Inputs[0]]: > Deps.append(self.PlaceMacro(str(F), self.Macros)) > # Add source-dependencies @@ -971,16 +991,63 @@ cleanlib: > if T.GenFileListMacro: > Deps.append("$(%s)" % T.FileListMacro) > if Type in [TAB_OBJECT_FILE, TAB_STATIC_LIBRARY]: > Deps.append("$(%s)" % T.ListFileMacro) > > - TargetDict = { > - "target" : self.PlaceMacro(T.Target.Path, self.Macros), > - "cmd" : "\n\t".join(T.Commands), > - "deps" : Deps > - } > - > self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(Target > Dic > t)) > + if self._AutoGenObject.BuildRuleFamily == > + TAB_COMPILER_MSFT and > Type == TAB_C_CODE_FILE: > + T, CmdTarget, CmdTargetDict, CmdCppDict = > + self.ParserCCodeFile(T, > Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict) > + TargetDict = {"target": > + self.PlaceMacro(T.Target.Path, self.Macros), > "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps} > + CmdLine = > self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$( > OBJLIS > T', '$(OBJLIST') > + if T.Commands: > + CmdLine = '%s%s' %(CmdLine, TAB_LINE_BREAK) > + if CCodeDeps or CmdLine: > + self.BuildTargetList.append(CmdLine) > + else: > + TargetDict = {"target": > + self.PlaceMacro(T.Target.Path, self.Macros), > "cmd": "\n\t".join(T.Commands),"deps": Deps} > + > + self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(Targ > + et > + Dict)) > + > + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, > + CmdCppDict, > DependencyDict): > + if not CmdSumDict: > + for item in self._AutoGenObject.Targets[Type]: > + CmdSumDict[item.Target.SubDir] = item.Target.BaseName > + for CppPath in item.Inputs: > + Path = self.PlaceMacro(CppPath.Path, self.Macros) > + if CmdCppDict.get(item.Target.SubDir): > + CmdCppDict[item.Target.SubDir].append(Path) > + else: > + CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path] > + if CppPath.Path in DependencyDict: > + for Temp in DependencyDict[CppPath.Path]: > + Path = self.PlaceMacro(Temp.Path, self.Macros) > + if Path not in (self.CommonFileDependency > + + > CmdCppDict[item.Target.SubDir]): > + CmdCppDict[item.Target.SubDir].append(Path) > + if T.Commands: > + CommandList = T.Commands[:] > + for Item in CommandList[:]: > + SingleCommandList = Item.split() > + if len(SingleCommandList) > 0 and '$(CC)' in SingleCommandList[0]: > + for Temp in SingleCommandList: > + if Temp.startswith('/Fo'): > + CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH) > + break > + else: continue > + if CmdSign not in list(CmdTargetDict.keys()): > + CmdLine = Item.replace(Temp, CmdSign) > + if GlobalData.gOptions.ThreadNumber: > + CmdLine = CmdLine.replace('/MP7', '%s%d' > + % ('/MP', > GlobalData.gOptions.ThreadNumber)) > + CmdTargetDict[CmdSign] = CmdLine > + else: > + CmdTargetDict[CmdSign] = "%s %s" % > + (CmdTargetDict[CmdSign], > SingleCommandList[-1]) > + Index = CommandList.index(Item) > + CommandList.pop(Index) > + if SingleCommandList[-1].endswith("%s%s.c" % > + (TAB_SLASH, > CmdSumDict[CmdSign.lstrip('/Fo').rsplit(TAB_SLASH, 1)[0]])): > + Cpplist = CmdCppDict[T.Target.SubDir] > + Cpplist.insert(0, '$(OBJLIST_%d): > + $(COMMON_DEPS)' % > list(self.ObjTargetDict.keys()).index(T.Target.SubDir)) > + T.Commands[Index] = '%s\n\t%s' % (' > + \\\n\t'.join(Cpplist), > CmdTargetDict[CmdSign]) > + else: > + T.Commands.pop(Index) > + return T, CmdSumDict, CmdTargetDict, CmdCppDict > > ## For creating makefile targets for dependent libraries > def ProcessDependentLibrary(self): > for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList: > if not LibraryAutoGen.IsBinaryModule: > -- > 2.20.1.windows.1 > > _______________________________________________ > edk2-devel mailing list > edk2-devel@lists.01.org > https://lists.01.org/mailman/listinfo/edk2-devel ^ permalink raw reply [flat|nested] 8+ messages in thread
end of thread, other threads:[~2019-04-03 2:45 UTC | newest] Thread overview: 8+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2019-03-29 12:30 [Patch] BaseTools:Enable the /MP option of MSVC compiler Feng, Bob C 2019-03-29 14:20 ` Gao, Liming 2019-04-03 2:11 ` Feng, Bob C 2019-04-03 2:45 ` Gao, Liming 2019-03-30 1:42 ` Ni, Ray 2019-03-30 8:14 ` Minnow Ware 2019-04-01 14:55 ` Feng, Bob C 2019-03-30 11:19 ` Feng, Bob C
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox