* [PATCH.V2] BaseTools: Limit command line length. @ 2020-11-05 1:26 mingyuex.liang 2020-11-05 2:43 ` Bob Feng 0 siblings, 1 reply; 6+ messages in thread From: mingyuex.liang @ 2020-11-05 1:26 UTC (permalink / raw) To: devel; +Cc: mliang2x, Bob Feng, Liming Gao, Yuwei Chen From: mliang2x <mingyuex.liang@intel.com> REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2528 Currently, CL command contains multiple C files will be compiled, and that caused command line too long, which may trigger build error. In order to solve this issue, the following rules is used in this scene: If the number of C files is greater than one, a txt file will be used to record these C files, and replaces the corresponding content in command line with the file name. Else (only one C file listed in the command line), the length of the whole CL command line will determine whether use a file to record. If the length exceeds the limited max length, use the recording file; else C file name directly listed in the command line V2: 1,Updated patch description 2,Changing the storage mode of C file is easy to obtain Signed-off-by: Mingyue Liang <mingyuex.liang@intel.com> Cc: Bob Feng <bob.c.feng@intel.com> Cc: Liming Gao <gaoliming@byosoft.com.cn> Cc: Yuwei Chen <yuwei.chen@intel.com> --- BaseTools/Source/Python/AutoGen/GenMake.py | 41 +++++++++++++++---- .../Source/Python/AutoGen/IncludesAutoGen.py | 12 +++++- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py index b04d3f54368e..1cfac1cd82ca 100755 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py @@ -576,7 +576,7 @@ cleanlib: EdkLogger.error("build", AUTOGEN_ERROR, "Nothing to build", ExtraData="[%s]" % str(MyAgo)) - self.ProcessBuildTargetList() + self.ProcessBuildTargetList(MyAgo.OutputDir, ToolsDef) self.ParserGenerateFfsCmd() # Generate macros used to represent input files @@ -903,7 +903,7 @@ cleanlib: BuildTargets[Target].Commands[i] = SingleCommand.replace('$(INC)', '').replace(FlagDict[Flag]['Macro'], RespMacro) return RespDict - def ProcessBuildTargetList(self): + def ProcessBuildTargetList(self, RespFile, ToolsDef): # # Search dependency file list for each source file # @@ -1004,6 +1004,7 @@ cleanlib: self.ObjTargetDict[T.Target.SubDir] = set() self.ObjTargetDict[T.Target.SubDir].add(NewFile) for Type in self._AutoGenObject.Targets: + resp_file_number = 0 for T in self._AutoGenObject.Targets[Type]: # Generate related macros if needed if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros: @@ -1045,7 +1046,10 @@ cleanlib: Deps.append("$(%s)" % T.ListFileMacro) if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE: - T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict) + T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, + CmdCppDict, DependencyDict, RespFile, + ToolsDef, resp_file_number) + resp_file_number += 1 TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps} CmdLine = self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIST', '$(OBJLIST') if T.Commands: @@ -1062,7 +1066,9 @@ cleanlib: AnnexeTargetDict = {"target": self.PlaceMacro(i.Path, self.Macros), "cmd": "", "deps": self.PlaceMacro(T.Target.Path, self.Macros)} self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(AnnexeTargetDict)) - def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict): + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict, RespFile, ToolsDef, + resp_file_number): + SaveFilePath = os.path.join(RespFile, "cc_resp_%s.txt" % resp_file_number) if not CmdSumDict: for item in self._AutoGenObject.Targets[Type]: CmdSumDict[item.Target.SubDir] = item.Target.BaseName @@ -1089,17 +1095,36 @@ cleanlib: if Temp.startswith('/Fo'): CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH) break - else: continue + else: + continue if CmdSign not in list(CmdTargetDict.keys()): - CmdTargetDict[CmdSign] = Item.replace(Temp, CmdSign) + cmd = Item.replace(Temp, CmdSign) + if SingleCommandList[-1] in cmd: + CmdTargetDict[CmdSign] = [cmd.replace(SingleCommandList[-1], "").rstrip(), SingleCommandList[-1]] else: - CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1]) + # CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1]) + CmdTargetDict[CmdSign].append(SingleCommandList[-1]) Index = CommandList.index(Item) CommandList.pop(Index) if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign[3:].rsplit(TAB_SLASH, 1)[0]])): Cpplist = CmdCppDict[T.Target.SubDir] Cpplist.insert(0, '$(OBJLIST_%d): ' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir)) - T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign]) + source_files = CmdTargetDict[CmdSign][1:] + source_files.insert(0, " ") + if len(source_files)>2: + SaveFileOnChange(SaveFilePath, " ".join(source_files), False) + T.Commands[Index] = '%s\n\t%s $(cc_resp_%s)' % ( + ' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign][0], resp_file_number) + ToolsDef.append("cc_resp_%s = @%s" % (resp_file_number, SaveFilePath)) + + elif len(source_files)<=2 and len(" ".join(CmdTargetDict[CmdSign][:2]))>GlobalData.gCommandMaxLength: + SaveFileOnChange(SaveFilePath, " ".join(source_files), False) + T.Commands[Index] = '%s\n\t%s $(cc_resp_%s)' % ( + ' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign][0], resp_file_number) + ToolsDef.append("cc_resp_%s = @%s" % (resp_file_number, SaveFilePath)) + + else: + T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), " ".join(CmdTargetDict[CmdSign])) else: T.Commands.pop(Index) return T, CmdSumDict, CmdTargetDict, CmdCppDict diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py index c3e6333217c4..b06ef42573fd 100644 --- a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py @@ -201,7 +201,17 @@ ${END} cc_options = line[len(cc_cmd)+2:].split() else: cc_options = line[len(cc_cmd):].split() - SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)} + for item in cc_options: + if not item.startswith("/"): + if item.endswith(".txt") and item.startswith("@"): + with open(item[1:], "r") as file: + source_files = file.readlines()[0].split() + SourceFileAbsPathMap = {os.path.basename(file): file for file in source_files if + os.path.exists(file)} + else: + if os.path.exists(item): + SourceFileAbsPathMap.update({os.path.basename(item): item.strip()}) + # SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)} if line in SourceFileAbsPathMap: current_source = line if current_source not in ModuleDepDict: -- 2.28.0.windows.1 ^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [PATCH.V2] BaseTools: Limit command line length. 2020-11-05 1:26 [PATCH.V2] BaseTools: Limit command line length mingyuex.liang @ 2020-11-05 2:43 ` Bob Feng 2020-11-05 6:33 ` Mingyue Liang 0 siblings, 1 reply; 6+ messages in thread From: Bob Feng @ 2020-11-05 2:43 UTC (permalink / raw) To: Liang, MingyueX, devel@edk2.groups.io Cc: Liang, MingyueX, Liming Gao, Chen, Christine, Vaish, Atul Hi Mingyue, Would you update the cc_resp_* file content? One c file one line. Thanks, Bob -----Original Message----- From: Mingyue Liang <mingyuex.liang@intel.com> Sent: Thursday, November 5, 2020 9:27 AM To: devel@edk2.groups.io Cc: Liang, MingyueX <mingyuex.liang@intel.com>; Feng, Bob C <bob.c.feng@intel.com>; Liming Gao <gaoliming@byosoft.com.cn>; Chen, Christine <yuwei.chen@intel.com> Subject: [PATCH.V2] BaseTools: Limit command line length. From: mliang2x <mingyuex.liang@intel.com> REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2528 Currently, CL command contains multiple C files will be compiled, and that caused command line too long, which may trigger build error. In order to solve this issue, the following rules is used in this scene: If the number of C files is greater than one, a txt file will be used to record these C files, and replaces the corresponding content in command line with the file name. Else (only one C file listed in the command line), the length of the whole CL command line will determine whether use a file to record. If the length exceeds the limited max length, use the recording file; else C file name directly listed in the command line V2: 1,Updated patch description 2,Changing the storage mode of C file is easy to obtain Signed-off-by: Mingyue Liang <mingyuex.liang@intel.com> Cc: Bob Feng <bob.c.feng@intel.com> Cc: Liming Gao <gaoliming@byosoft.com.cn> Cc: Yuwei Chen <yuwei.chen@intel.com> --- BaseTools/Source/Python/AutoGen/GenMake.py | 41 +++++++++++++++---- .../Source/Python/AutoGen/IncludesAutoGen.py | 12 +++++- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py index b04d3f54368e..1cfac1cd82ca 100755 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py @@ -576,7 +576,7 @@ cleanlib: EdkLogger.error("build", AUTOGEN_ERROR, "Nothing to build", ExtraData="[%s]" % str(MyAgo)) - self.ProcessBuildTargetList() + self.ProcessBuildTargetList(MyAgo.OutputDir, ToolsDef) self.ParserGenerateFfsCmd() # Generate macros used to represent input files @@ -903,7 +903,7 @@ cleanlib: BuildTargets[Target].Commands[i] = SingleCommand.replace('$(INC)', '').replace(FlagDict[Flag]['Macro'], RespMacro) return RespDict - def ProcessBuildTargetList(self): + def ProcessBuildTargetList(self, RespFile, ToolsDef): # # Search dependency file list for each source file # @@ -1004,6 +1004,7 @@ cleanlib: self.ObjTargetDict[T.Target.SubDir] = set() self.ObjTargetDict[T.Target.SubDir].add(NewFile) for Type in self._AutoGenObject.Targets: + resp_file_number = 0 for T in self._AutoGenObject.Targets[Type]: # Generate related macros if needed if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros: @@ -1045,7 +1046,10 @@ cleanlib: Deps.append("$(%s)" % T.ListFileMacro) if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE: - T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict) + T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, + CmdCppDict, DependencyDict, RespFile, + ToolsDef, resp_file_number) + resp_file_number += 1 TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps} CmdLine = self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIST', '$(OBJLIST') if T.Commands: @@ -1062,7 +1066,9 @@ cleanlib: AnnexeTargetDict = {"target": self.PlaceMacro(i.Path, self.Macros), "cmd": "", "deps": self.PlaceMacro(T.Target.Path, self.Macros)} self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(AnnexeTargetDict)) - def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict): + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict, RespFile, ToolsDef, + resp_file_number): + SaveFilePath = os.path.join(RespFile, "cc_resp_%s.txt" % + resp_file_number) if not CmdSumDict: for item in self._AutoGenObject.Targets[Type]: CmdSumDict[item.Target.SubDir] = item.Target.BaseName @@ -1089,17 +1095,36 @@ cleanlib: if Temp.startswith('/Fo'): CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH) break - else: continue + else: + continue if CmdSign not in list(CmdTargetDict.keys()): - CmdTargetDict[CmdSign] = Item.replace(Temp, CmdSign) + cmd = Item.replace(Temp, CmdSign) + if SingleCommandList[-1] in cmd: + CmdTargetDict[CmdSign] = + [cmd.replace(SingleCommandList[-1], "").rstrip(), + SingleCommandList[-1]] else: - CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1]) + # CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1]) + + CmdTargetDict[CmdSign].append(SingleCommandList[-1]) Index = CommandList.index(Item) CommandList.pop(Index) if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign[3:].rsplit(TAB_SLASH, 1)[0]])): Cpplist = CmdCppDict[T.Target.SubDir] Cpplist.insert(0, '$(OBJLIST_%d): ' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir)) - T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign]) + source_files = CmdTargetDict[CmdSign][1:] + source_files.insert(0, " ") + if len(source_files)>2: + SaveFileOnChange(SaveFilePath, " ".join(source_files), False) + T.Commands[Index] = '%s\n\t%s $(cc_resp_%s)' % ( + ' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign][0], resp_file_number) + ToolsDef.append("cc_resp_%s = @%s" % + (resp_file_number, SaveFilePath)) + + elif len(source_files)<=2 and len(" ".join(CmdTargetDict[CmdSign][:2]))>GlobalData.gCommandMaxLength: + SaveFileOnChange(SaveFilePath, " ".join(source_files), False) + T.Commands[Index] = '%s\n\t%s $(cc_resp_%s)' % ( + ' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign][0], resp_file_number) + ToolsDef.append("cc_resp_%s = @%s" % + (resp_file_number, SaveFilePath)) + + else: + T.Commands[Index] = '%s\n\t%s' % (' + \\\n\t'.join(Cpplist), " ".join(CmdTargetDict[CmdSign])) else: T.Commands.pop(Index) return T, CmdSumDict, CmdTargetDict, CmdCppDict diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py index c3e6333217c4..b06ef42573fd 100644 --- a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py @@ -201,7 +201,17 @@ ${END} cc_options = line[len(cc_cmd)+2:].split() else: cc_options = line[len(cc_cmd):].split() - SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)} + for item in cc_options: + if not item.startswith("/"): + if item.endswith(".txt") and item.startswith("@"): + with open(item[1:], "r") as file: + source_files = file.readlines()[0].split() + SourceFileAbsPathMap = {os.path.basename(file): file for file in source_files if + os.path.exists(file)} + else: + if os.path.exists(item): + SourceFileAbsPathMap.update({os.path.basename(item): item.strip()}) + # SourceFileAbsPathMap = + {os.path.basename(item):item for item in cc_options if not + item.startswith("/") and os.path.exists(item)} if line in SourceFileAbsPathMap: current_source = line if current_source not in ModuleDepDict: -- 2.28.0.windows.1 ^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [PATCH.V2] BaseTools: Limit command line length. 2020-11-05 2:43 ` Bob Feng @ 2020-11-05 6:33 ` Mingyue Liang 2020-11-09 6:20 ` Bob Feng [not found] ` <1645C2A4018343CE.24110@groups.io> 0 siblings, 2 replies; 6+ messages in thread From: Mingyue Liang @ 2020-11-05 6:33 UTC (permalink / raw) To: Feng, Bob C, devel@edk2.groups.io Cc: Liming Gao, Chen, Christine, Vaish, Atul Hi Bob, This method is not recommended and will result in build errors. Thanks, Mingyue -----Original Message----- From: Feng, Bob C <bob.c.feng@intel.com> Sent: Thursday, November 5, 2020 10:43 AM To: Liang, MingyueX <mingyuex.liang@intel.com>; devel@edk2.groups.io Cc: Liang, MingyueX <mingyuex.liang@intel.com>; Liming Gao <gaoliming@byosoft.com.cn>; Chen, Christine <yuwei.chen@intel.com>; Vaish, Atul <atul.vaish@intel.com> Subject: RE: [PATCH.V2] BaseTools: Limit command line length. Hi Mingyue, Would you update the cc_resp_* file content? One c file one line. Thanks, Bob -----Original Message----- From: Mingyue Liang <mingyuex.liang@intel.com> Sent: Thursday, November 5, 2020 9:27 AM To: devel@edk2.groups.io Cc: Liang, MingyueX <mingyuex.liang@intel.com>; Feng, Bob C <bob.c.feng@intel.com>; Liming Gao <gaoliming@byosoft.com.cn>; Chen, Christine <yuwei.chen@intel.com> Subject: [PATCH.V2] BaseTools: Limit command line length. From: mliang2x <mingyuex.liang@intel.com> REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2528 Currently, CL command contains multiple C files will be compiled, and that caused command line too long, which may trigger build error. In order to solve this issue, the following rules is used in this scene: If the number of C files is greater than one, a txt file will be used to record these C files, and replaces the corresponding content in command line with the file name. Else (only one C file listed in the command line), the length of the whole CL command line will determine whether use a file to record. If the length exceeds the limited max length, use the recording file; else C file name directly listed in the command line V2: 1,Updated patch description 2,Changing the storage mode of C file is easy to obtain Signed-off-by: Mingyue Liang <mingyuex.liang@intel.com> Cc: Bob Feng <bob.c.feng@intel.com> Cc: Liming Gao <gaoliming@byosoft.com.cn> Cc: Yuwei Chen <yuwei.chen@intel.com> --- BaseTools/Source/Python/AutoGen/GenMake.py | 41 +++++++++++++++---- .../Source/Python/AutoGen/IncludesAutoGen.py | 12 +++++- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py index b04d3f54368e..1cfac1cd82ca 100755 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py @@ -576,7 +576,7 @@ cleanlib: EdkLogger.error("build", AUTOGEN_ERROR, "Nothing to build", ExtraData="[%s]" % str(MyAgo)) - self.ProcessBuildTargetList() + self.ProcessBuildTargetList(MyAgo.OutputDir, ToolsDef) self.ParserGenerateFfsCmd() # Generate macros used to represent input files @@ -903,7 +903,7 @@ cleanlib: BuildTargets[Target].Commands[i] = SingleCommand.replace('$(INC)', '').replace(FlagDict[Flag]['Macro'], RespMacro) return RespDict - def ProcessBuildTargetList(self): + def ProcessBuildTargetList(self, RespFile, ToolsDef): # # Search dependency file list for each source file # @@ -1004,6 +1004,7 @@ cleanlib: self.ObjTargetDict[T.Target.SubDir] = set() self.ObjTargetDict[T.Target.SubDir].add(NewFile) for Type in self._AutoGenObject.Targets: + resp_file_number = 0 for T in self._AutoGenObject.Targets[Type]: # Generate related macros if needed if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros: @@ -1045,7 +1046,10 @@ cleanlib: Deps.append("$(%s)" % T.ListFileMacro) if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE: - T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict) + T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, + CmdCppDict, DependencyDict, RespFile, + ToolsDef, resp_file_number) + resp_file_number += 1 TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps} CmdLine = self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIST', '$(OBJLIST') if T.Commands: @@ -1062,7 +1066,9 @@ cleanlib: AnnexeTargetDict = {"target": self.PlaceMacro(i.Path, self.Macros), "cmd": "", "deps": self.PlaceMacro(T.Target.Path, self.Macros)} self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(AnnexeTargetDict)) - def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict): + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict, RespFile, ToolsDef, + resp_file_number): + SaveFilePath = os.path.join(RespFile, "cc_resp_%s.txt" % + resp_file_number) if not CmdSumDict: for item in self._AutoGenObject.Targets[Type]: CmdSumDict[item.Target.SubDir] = item.Target.BaseName @@ -1089,17 +1095,36 @@ cleanlib: if Temp.startswith('/Fo'): CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH) break - else: continue + else: + continue if CmdSign not in list(CmdTargetDict.keys()): - CmdTargetDict[CmdSign] = Item.replace(Temp, CmdSign) + cmd = Item.replace(Temp, CmdSign) + if SingleCommandList[-1] in cmd: + CmdTargetDict[CmdSign] = + [cmd.replace(SingleCommandList[-1], "").rstrip(), + SingleCommandList[-1]] else: - CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1]) + # CmdTargetDict[CmdSign] = "%s %s" % + (CmdTargetDict[CmdSign], SingleCommandList[-1]) + + CmdTargetDict[CmdSign].append(SingleCommandList[-1]) Index = CommandList.index(Item) CommandList.pop(Index) if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign[3:].rsplit(TAB_SLASH, 1)[0]])): Cpplist = CmdCppDict[T.Target.SubDir] Cpplist.insert(0, '$(OBJLIST_%d): ' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir)) - T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign]) + source_files = CmdTargetDict[CmdSign][1:] + source_files.insert(0, " ") + if len(source_files)>2: + SaveFileOnChange(SaveFilePath, " ".join(source_files), False) + T.Commands[Index] = '%s\n\t%s $(cc_resp_%s)' % ( + ' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign][0], resp_file_number) + ToolsDef.append("cc_resp_%s = @%s" % + (resp_file_number, SaveFilePath)) + + elif len(source_files)<=2 and len(" ".join(CmdTargetDict[CmdSign][:2]))>GlobalData.gCommandMaxLength: + SaveFileOnChange(SaveFilePath, " ".join(source_files), False) + T.Commands[Index] = '%s\n\t%s $(cc_resp_%s)' % ( + ' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign][0], resp_file_number) + ToolsDef.append("cc_resp_%s = @%s" % + (resp_file_number, SaveFilePath)) + + else: + T.Commands[Index] = '%s\n\t%s' % (' + \\\n\t'.join(Cpplist), " ".join(CmdTargetDict[CmdSign])) else: T.Commands.pop(Index) return T, CmdSumDict, CmdTargetDict, CmdCppDict diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py index c3e6333217c4..b06ef42573fd 100644 --- a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py @@ -201,7 +201,17 @@ ${END} cc_options = line[len(cc_cmd)+2:].split() else: cc_options = line[len(cc_cmd):].split() - SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)} + for item in cc_options: + if not item.startswith("/"): + if item.endswith(".txt") and item.startswith("@"): + with open(item[1:], "r") as file: + source_files = file.readlines()[0].split() + SourceFileAbsPathMap = {os.path.basename(file): file for file in source_files if + os.path.exists(file)} + else: + if os.path.exists(item): + SourceFileAbsPathMap.update({os.path.basename(item): item.strip()}) + # SourceFileAbsPathMap = + {os.path.basename(item):item for item in cc_options if not + item.startswith("/") and os.path.exists(item)} if line in SourceFileAbsPathMap: current_source = line if current_source not in ModuleDepDict: -- 2.28.0.windows.1 ^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [PATCH.V2] BaseTools: Limit command line length. 2020-11-05 6:33 ` Mingyue Liang @ 2020-11-09 6:20 ` Bob Feng [not found] ` <1645C2A4018343CE.24110@groups.io> 1 sibling, 0 replies; 6+ messages in thread From: Bob Feng @ 2020-11-09 6:20 UTC (permalink / raw) To: Liang, MingyueX, devel@edk2.groups.io Cc: Liming Gao, Chen, Christine, Vaish, Atul OK. Reviewed-by: Bob Feng <bob.c.feng@intel.com> -----Original Message----- From: Liang, MingyueX <mingyuex.liang@intel.com> Sent: Thursday, November 5, 2020 2:33 PM To: Feng, Bob C <bob.c.feng@intel.com>; devel@edk2.groups.io Cc: Liming Gao <gaoliming@byosoft.com.cn>; Chen, Christine <yuwei.chen@intel.com>; Vaish, Atul <atul.vaish@intel.com> Subject: RE: [PATCH.V2] BaseTools: Limit command line length. Hi Bob, This method is not recommended and will result in build errors. Thanks, Mingyue -----Original Message----- From: Feng, Bob C <bob.c.feng@intel.com> Sent: Thursday, November 5, 2020 10:43 AM To: Liang, MingyueX <mingyuex.liang@intel.com>; devel@edk2.groups.io Cc: Liang, MingyueX <mingyuex.liang@intel.com>; Liming Gao <gaoliming@byosoft.com.cn>; Chen, Christine <yuwei.chen@intel.com>; Vaish, Atul <atul.vaish@intel.com> Subject: RE: [PATCH.V2] BaseTools: Limit command line length. Hi Mingyue, Would you update the cc_resp_* file content? One c file one line. Thanks, Bob -----Original Message----- From: Mingyue Liang <mingyuex.liang@intel.com> Sent: Thursday, November 5, 2020 9:27 AM To: devel@edk2.groups.io Cc: Liang, MingyueX <mingyuex.liang@intel.com>; Feng, Bob C <bob.c.feng@intel.com>; Liming Gao <gaoliming@byosoft.com.cn>; Chen, Christine <yuwei.chen@intel.com> Subject: [PATCH.V2] BaseTools: Limit command line length. From: mliang2x <mingyuex.liang@intel.com> REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2528 Currently, CL command contains multiple C files will be compiled, and that caused command line too long, which may trigger build error. In order to solve this issue, the following rules is used in this scene: If the number of C files is greater than one, a txt file will be used to record these C files, and replaces the corresponding content in command line with the file name. Else (only one C file listed in the command line), the length of the whole CL command line will determine whether use a file to record. If the length exceeds the limited max length, use the recording file; else C file name directly listed in the command line V2: 1,Updated patch description 2,Changing the storage mode of C file is easy to obtain Signed-off-by: Mingyue Liang <mingyuex.liang@intel.com> Cc: Bob Feng <bob.c.feng@intel.com> Cc: Liming Gao <gaoliming@byosoft.com.cn> Cc: Yuwei Chen <yuwei.chen@intel.com> --- BaseTools/Source/Python/AutoGen/GenMake.py | 41 +++++++++++++++---- .../Source/Python/AutoGen/IncludesAutoGen.py | 12 +++++- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py index b04d3f54368e..1cfac1cd82ca 100755 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py @@ -576,7 +576,7 @@ cleanlib: EdkLogger.error("build", AUTOGEN_ERROR, "Nothing to build", ExtraData="[%s]" % str(MyAgo)) - self.ProcessBuildTargetList() + self.ProcessBuildTargetList(MyAgo.OutputDir, ToolsDef) self.ParserGenerateFfsCmd() # Generate macros used to represent input files @@ -903,7 +903,7 @@ cleanlib: BuildTargets[Target].Commands[i] = SingleCommand.replace('$(INC)', '').replace(FlagDict[Flag]['Macro'], RespMacro) return RespDict - def ProcessBuildTargetList(self): + def ProcessBuildTargetList(self, RespFile, ToolsDef): # # Search dependency file list for each source file # @@ -1004,6 +1004,7 @@ cleanlib: self.ObjTargetDict[T.Target.SubDir] = set() self.ObjTargetDict[T.Target.SubDir].add(NewFile) for Type in self._AutoGenObject.Targets: + resp_file_number = 0 for T in self._AutoGenObject.Targets[Type]: # Generate related macros if needed if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros: @@ -1045,7 +1046,10 @@ cleanlib: Deps.append("$(%s)" % T.ListFileMacro) if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE: - T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict) + T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, + CmdCppDict, DependencyDict, RespFile, + ToolsDef, resp_file_number) + resp_file_number += 1 TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps} CmdLine = self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIST', '$(OBJLIST') if T.Commands: @@ -1062,7 +1066,9 @@ cleanlib: AnnexeTargetDict = {"target": self.PlaceMacro(i.Path, self.Macros), "cmd": "", "deps": self.PlaceMacro(T.Target.Path, self.Macros)} self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(AnnexeTargetDict)) - def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict): + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict, RespFile, ToolsDef, + resp_file_number): + SaveFilePath = os.path.join(RespFile, "cc_resp_%s.txt" % + resp_file_number) if not CmdSumDict: for item in self._AutoGenObject.Targets[Type]: CmdSumDict[item.Target.SubDir] = item.Target.BaseName @@ -1089,17 +1095,36 @@ cleanlib: if Temp.startswith('/Fo'): CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH) break - else: continue + else: + continue if CmdSign not in list(CmdTargetDict.keys()): - CmdTargetDict[CmdSign] = Item.replace(Temp, CmdSign) + cmd = Item.replace(Temp, CmdSign) + if SingleCommandList[-1] in cmd: + CmdTargetDict[CmdSign] = + [cmd.replace(SingleCommandList[-1], "").rstrip(), + SingleCommandList[-1]] else: - CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1]) + # CmdTargetDict[CmdSign] = "%s %s" % + (CmdTargetDict[CmdSign], SingleCommandList[-1]) + + CmdTargetDict[CmdSign].append(SingleCommandList[-1]) Index = CommandList.index(Item) CommandList.pop(Index) if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign[3:].rsplit(TAB_SLASH, 1)[0]])): Cpplist = CmdCppDict[T.Target.SubDir] Cpplist.insert(0, '$(OBJLIST_%d): ' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir)) - T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign]) + source_files = CmdTargetDict[CmdSign][1:] + source_files.insert(0, " ") + if len(source_files)>2: + SaveFileOnChange(SaveFilePath, " ".join(source_files), False) + T.Commands[Index] = '%s\n\t%s $(cc_resp_%s)' % ( + ' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign][0], resp_file_number) + ToolsDef.append("cc_resp_%s = @%s" % + (resp_file_number, SaveFilePath)) + + elif len(source_files)<=2 and len(" ".join(CmdTargetDict[CmdSign][:2]))>GlobalData.gCommandMaxLength: + SaveFileOnChange(SaveFilePath, " ".join(source_files), False) + T.Commands[Index] = '%s\n\t%s $(cc_resp_%s)' % ( + ' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign][0], resp_file_number) + ToolsDef.append("cc_resp_%s = @%s" % + (resp_file_number, SaveFilePath)) + + else: + T.Commands[Index] = '%s\n\t%s' % (' + \\\n\t'.join(Cpplist), " ".join(CmdTargetDict[CmdSign])) else: T.Commands.pop(Index) return T, CmdSumDict, CmdTargetDict, CmdCppDict diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py index c3e6333217c4..b06ef42573fd 100644 --- a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py @@ -201,7 +201,17 @@ ${END} cc_options = line[len(cc_cmd)+2:].split() else: cc_options = line[len(cc_cmd):].split() - SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)} + for item in cc_options: + if not item.startswith("/"): + if item.endswith(".txt") and item.startswith("@"): + with open(item[1:], "r") as file: + source_files = file.readlines()[0].split() + SourceFileAbsPathMap = {os.path.basename(file): file for file in source_files if + os.path.exists(file)} + else: + if os.path.exists(item): + SourceFileAbsPathMap.update({os.path.basename(item): item.strip()}) + # SourceFileAbsPathMap = + {os.path.basename(item):item for item in cc_options if not + item.startswith("/") and os.path.exists(item)} if line in SourceFileAbsPathMap: current_source = line if current_source not in ModuleDepDict: -- 2.28.0.windows.1 ^ permalink raw reply related [flat|nested] 6+ messages in thread
[parent not found: <1645C2A4018343CE.24110@groups.io>]
* Re: [edk2-devel] [PATCH.V2] BaseTools: Limit command line length. [not found] ` <1645C2A4018343CE.24110@groups.io> @ 2020-11-10 4:10 ` Bob Feng 2020-11-10 5:44 ` Mingyue Liang 0 siblings, 1 reply; 6+ messages in thread From: Bob Feng @ 2020-11-10 4:10 UTC (permalink / raw) To: devel@edk2.groups.io, Feng, Bob C, Liang, MingyueX Cc: Liming Gao, Chen, Christine, Vaish, Atul Mingyue, Please remove the statements of "V2: ..." from the commit message. Thanks, Bob -----Original Message----- From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Bob Feng Sent: Monday, November 9, 2020 2:21 PM To: Liang, MingyueX <mingyuex.liang@intel.com>; devel@edk2.groups.io Cc: Liming Gao <gaoliming@byosoft.com.cn>; Chen, Christine <yuwei.chen@intel.com>; Vaish, Atul <atul.vaish@intel.com> Subject: Re: [edk2-devel] [PATCH.V2] BaseTools: Limit command line length. OK. Reviewed-by: Bob Feng <bob.c.feng@intel.com> -----Original Message----- From: Liang, MingyueX <mingyuex.liang@intel.com> Sent: Thursday, November 5, 2020 2:33 PM To: Feng, Bob C <bob.c.feng@intel.com>; devel@edk2.groups.io Cc: Liming Gao <gaoliming@byosoft.com.cn>; Chen, Christine <yuwei.chen@intel.com>; Vaish, Atul <atul.vaish@intel.com> Subject: RE: [PATCH.V2] BaseTools: Limit command line length. Hi Bob, This method is not recommended and will result in build errors. Thanks, Mingyue -----Original Message----- From: Feng, Bob C <bob.c.feng@intel.com> Sent: Thursday, November 5, 2020 10:43 AM To: Liang, MingyueX <mingyuex.liang@intel.com>; devel@edk2.groups.io Cc: Liang, MingyueX <mingyuex.liang@intel.com>; Liming Gao <gaoliming@byosoft.com.cn>; Chen, Christine <yuwei.chen@intel.com>; Vaish, Atul <atul.vaish@intel.com> Subject: RE: [PATCH.V2] BaseTools: Limit command line length. Hi Mingyue, Would you update the cc_resp_* file content? One c file one line. Thanks, Bob -----Original Message----- From: Mingyue Liang <mingyuex.liang@intel.com> Sent: Thursday, November 5, 2020 9:27 AM To: devel@edk2.groups.io Cc: Liang, MingyueX <mingyuex.liang@intel.com>; Feng, Bob C <bob.c.feng@intel.com>; Liming Gao <gaoliming@byosoft.com.cn>; Chen, Christine <yuwei.chen@intel.com> Subject: [PATCH.V2] BaseTools: Limit command line length. From: mliang2x <mingyuex.liang@intel.com> REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2528 Currently, CL command contains multiple C files will be compiled, and that caused command line too long, which may trigger build error. In order to solve this issue, the following rules is used in this scene: If the number of C files is greater than one, a txt file will be used to record these C files, and replaces the corresponding content in command line with the file name. Else (only one C file listed in the command line), the length of the whole CL command line will determine whether use a file to record. If the length exceeds the limited max length, use the recording file; else C file name directly listed in the command line V2: 1,Updated patch description 2,Changing the storage mode of C file is easy to obtain Signed-off-by: Mingyue Liang <mingyuex.liang@intel.com> Cc: Bob Feng <bob.c.feng@intel.com> Cc: Liming Gao <gaoliming@byosoft.com.cn> Cc: Yuwei Chen <yuwei.chen@intel.com> --- BaseTools/Source/Python/AutoGen/GenMake.py | 41 +++++++++++++++---- .../Source/Python/AutoGen/IncludesAutoGen.py | 12 +++++- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py index b04d3f54368e..1cfac1cd82ca 100755 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py @@ -576,7 +576,7 @@ cleanlib: EdkLogger.error("build", AUTOGEN_ERROR, "Nothing to build", ExtraData="[%s]" % str(MyAgo)) - self.ProcessBuildTargetList() + self.ProcessBuildTargetList(MyAgo.OutputDir, ToolsDef) self.ParserGenerateFfsCmd() # Generate macros used to represent input files @@ -903,7 +903,7 @@ cleanlib: BuildTargets[Target].Commands[i] = SingleCommand.replace('$(INC)', '').replace(FlagDict[Flag]['Macro'], RespMacro) return RespDict - def ProcessBuildTargetList(self): + def ProcessBuildTargetList(self, RespFile, ToolsDef): # # Search dependency file list for each source file # @@ -1004,6 +1004,7 @@ cleanlib: self.ObjTargetDict[T.Target.SubDir] = set() self.ObjTargetDict[T.Target.SubDir].add(NewFile) for Type in self._AutoGenObject.Targets: + resp_file_number = 0 for T in self._AutoGenObject.Targets[Type]: # Generate related macros if needed if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros: @@ -1045,7 +1046,10 @@ cleanlib: Deps.append("$(%s)" % T.ListFileMacro) if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE: - T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict) + T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, + CmdCppDict, DependencyDict, RespFile, + ToolsDef, resp_file_number) + resp_file_number += 1 TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps} CmdLine = self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIST', '$(OBJLIST') if T.Commands: @@ -1062,7 +1066,9 @@ cleanlib: AnnexeTargetDict = {"target": self.PlaceMacro(i.Path, self.Macros), "cmd": "", "deps": self.PlaceMacro(T.Target.Path, self.Macros)} self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(AnnexeTargetDict)) - def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict): + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict, RespFile, ToolsDef, + resp_file_number): + SaveFilePath = os.path.join(RespFile, "cc_resp_%s.txt" % + resp_file_number) if not CmdSumDict: for item in self._AutoGenObject.Targets[Type]: CmdSumDict[item.Target.SubDir] = item.Target.BaseName @@ -1089,17 +1095,36 @@ cleanlib: if Temp.startswith('/Fo'): CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH) break - else: continue + else: + continue if CmdSign not in list(CmdTargetDict.keys()): - CmdTargetDict[CmdSign] = Item.replace(Temp, CmdSign) + cmd = Item.replace(Temp, CmdSign) + if SingleCommandList[-1] in cmd: + CmdTargetDict[CmdSign] = + [cmd.replace(SingleCommandList[-1], "").rstrip(), + SingleCommandList[-1]] else: - CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1]) + # CmdTargetDict[CmdSign] = "%s %s" % + (CmdTargetDict[CmdSign], SingleCommandList[-1]) + + CmdTargetDict[CmdSign].append(SingleCommandList[-1]) Index = CommandList.index(Item) CommandList.pop(Index) if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign[3:].rsplit(TAB_SLASH, 1)[0]])): Cpplist = CmdCppDict[T.Target.SubDir] Cpplist.insert(0, '$(OBJLIST_%d): ' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir)) - T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign]) + source_files = CmdTargetDict[CmdSign][1:] + source_files.insert(0, " ") + if len(source_files)>2: + SaveFileOnChange(SaveFilePath, " ".join(source_files), False) + T.Commands[Index] = '%s\n\t%s $(cc_resp_%s)' % ( + ' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign][0], resp_file_number) + ToolsDef.append("cc_resp_%s = @%s" % + (resp_file_number, SaveFilePath)) + + elif len(source_files)<=2 and len(" ".join(CmdTargetDict[CmdSign][:2]))>GlobalData.gCommandMaxLength: + SaveFileOnChange(SaveFilePath, " ".join(source_files), False) + T.Commands[Index] = '%s\n\t%s $(cc_resp_%s)' % ( + ' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign][0], resp_file_number) + ToolsDef.append("cc_resp_%s = @%s" % + (resp_file_number, SaveFilePath)) + + else: + T.Commands[Index] = '%s\n\t%s' % (' + \\\n\t'.join(Cpplist), " ".join(CmdTargetDict[CmdSign])) else: T.Commands.pop(Index) return T, CmdSumDict, CmdTargetDict, CmdCppDict diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py index c3e6333217c4..b06ef42573fd 100644 --- a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py @@ -201,7 +201,17 @@ ${END} cc_options = line[len(cc_cmd)+2:].split() else: cc_options = line[len(cc_cmd):].split() - SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)} + for item in cc_options: + if not item.startswith("/"): + if item.endswith(".txt") and item.startswith("@"): + with open(item[1:], "r") as file: + source_files = file.readlines()[0].split() + SourceFileAbsPathMap = {os.path.basename(file): file for file in source_files if + os.path.exists(file)} + else: + if os.path.exists(item): + SourceFileAbsPathMap.update({os.path.basename(item): item.strip()}) + # SourceFileAbsPathMap = + {os.path.basename(item):item for item in cc_options if not + item.startswith("/") and os.path.exists(item)} if line in SourceFileAbsPathMap: current_source = line if current_source not in ModuleDepDict: -- 2.28.0.windows.1 ^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [edk2-devel] [PATCH.V2] BaseTools: Limit command line length. 2020-11-10 4:10 ` [edk2-devel] " Bob Feng @ 2020-11-10 5:44 ` Mingyue Liang 0 siblings, 0 replies; 6+ messages in thread From: Mingyue Liang @ 2020-11-10 5:44 UTC (permalink / raw) To: Feng, Bob C, devel@edk2.groups.io Cc: Liming Gao, Chen, Christine, Vaish, Atul Updated to V3. -----Original Message----- From: Feng, Bob C <bob.c.feng@intel.com> Sent: Tuesday, November 10, 2020 12:10 PM To: devel@edk2.groups.io; Feng, Bob C <bob.c.feng@intel.com>; Liang, MingyueX <mingyuex.liang@intel.com> Cc: Liming Gao <gaoliming@byosoft.com.cn>; Chen, Christine <yuwei.chen@intel.com>; Vaish, Atul <atul.vaish@intel.com> Subject: RE: [edk2-devel] [PATCH.V2] BaseTools: Limit command line length. Mingyue, Please remove the statements of "V2: ..." from the commit message. Thanks, Bob -----Original Message----- From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Bob Feng Sent: Monday, November 9, 2020 2:21 PM To: Liang, MingyueX <mingyuex.liang@intel.com>; devel@edk2.groups.io Cc: Liming Gao <gaoliming@byosoft.com.cn>; Chen, Christine <yuwei.chen@intel.com>; Vaish, Atul <atul.vaish@intel.com> Subject: Re: [edk2-devel] [PATCH.V2] BaseTools: Limit command line length. OK. Reviewed-by: Bob Feng <bob.c.feng@intel.com> -----Original Message----- From: Liang, MingyueX <mingyuex.liang@intel.com> Sent: Thursday, November 5, 2020 2:33 PM To: Feng, Bob C <bob.c.feng@intel.com>; devel@edk2.groups.io Cc: Liming Gao <gaoliming@byosoft.com.cn>; Chen, Christine <yuwei.chen@intel.com>; Vaish, Atul <atul.vaish@intel.com> Subject: RE: [PATCH.V2] BaseTools: Limit command line length. Hi Bob, This method is not recommended and will result in build errors. Thanks, Mingyue -----Original Message----- From: Feng, Bob C <bob.c.feng@intel.com> Sent: Thursday, November 5, 2020 10:43 AM To: Liang, MingyueX <mingyuex.liang@intel.com>; devel@edk2.groups.io Cc: Liang, MingyueX <mingyuex.liang@intel.com>; Liming Gao <gaoliming@byosoft.com.cn>; Chen, Christine <yuwei.chen@intel.com>; Vaish, Atul <atul.vaish@intel.com> Subject: RE: [PATCH.V2] BaseTools: Limit command line length. Hi Mingyue, Would you update the cc_resp_* file content? One c file one line. Thanks, Bob -----Original Message----- From: Mingyue Liang <mingyuex.liang@intel.com> Sent: Thursday, November 5, 2020 9:27 AM To: devel@edk2.groups.io Cc: Liang, MingyueX <mingyuex.liang@intel.com>; Feng, Bob C <bob.c.feng@intel.com>; Liming Gao <gaoliming@byosoft.com.cn>; Chen, Christine <yuwei.chen@intel.com> Subject: [PATCH.V2] BaseTools: Limit command line length. From: mliang2x <mingyuex.liang@intel.com> REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2528 Currently, CL command contains multiple C files will be compiled, and that caused command line too long, which may trigger build error. In order to solve this issue, the following rules is used in this scene: If the number of C files is greater than one, a txt file will be used to record these C files, and replaces the corresponding content in command line with the file name. Else (only one C file listed in the command line), the length of the whole CL command line will determine whether use a file to record. If the length exceeds the limited max length, use the recording file; else C file name directly listed in the command line V2: 1,Updated patch description 2,Changing the storage mode of C file is easy to obtain Signed-off-by: Mingyue Liang <mingyuex.liang@intel.com> Cc: Bob Feng <bob.c.feng@intel.com> Cc: Liming Gao <gaoliming@byosoft.com.cn> Cc: Yuwei Chen <yuwei.chen@intel.com> --- BaseTools/Source/Python/AutoGen/GenMake.py | 41 +++++++++++++++---- .../Source/Python/AutoGen/IncludesAutoGen.py | 12 +++++- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py index b04d3f54368e..1cfac1cd82ca 100755 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py @@ -576,7 +576,7 @@ cleanlib: EdkLogger.error("build", AUTOGEN_ERROR, "Nothing to build", ExtraData="[%s]" % str(MyAgo)) - self.ProcessBuildTargetList() + self.ProcessBuildTargetList(MyAgo.OutputDir, ToolsDef) self.ParserGenerateFfsCmd() # Generate macros used to represent input files @@ -903,7 +903,7 @@ cleanlib: BuildTargets[Target].Commands[i] = SingleCommand.replace('$(INC)', '').replace(FlagDict[Flag]['Macro'], RespMacro) return RespDict - def ProcessBuildTargetList(self): + def ProcessBuildTargetList(self, RespFile, ToolsDef): # # Search dependency file list for each source file # @@ -1004,6 +1004,7 @@ cleanlib: self.ObjTargetDict[T.Target.SubDir] = set() self.ObjTargetDict[T.Target.SubDir].add(NewFile) for Type in self._AutoGenObject.Targets: + resp_file_number = 0 for T in self._AutoGenObject.Targets[Type]: # Generate related macros if needed if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros: @@ -1045,7 +1046,10 @@ cleanlib: Deps.append("$(%s)" % T.ListFileMacro) if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE: - T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict) + T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, + CmdCppDict, DependencyDict, RespFile, + ToolsDef, resp_file_number) + resp_file_number += 1 TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps} CmdLine = self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIST', '$(OBJLIST') if T.Commands: @@ -1062,7 +1066,9 @@ cleanlib: AnnexeTargetDict = {"target": self.PlaceMacro(i.Path, self.Macros), "cmd": "", "deps": self.PlaceMacro(T.Target.Path, self.Macros)} self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(AnnexeTargetDict)) - def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict): + def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict, RespFile, ToolsDef, + resp_file_number): + SaveFilePath = os.path.join(RespFile, "cc_resp_%s.txt" % + resp_file_number) if not CmdSumDict: for item in self._AutoGenObject.Targets[Type]: CmdSumDict[item.Target.SubDir] = item.Target.BaseName @@ -1089,17 +1095,36 @@ cleanlib: if Temp.startswith('/Fo'): CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH) break - else: continue + else: + continue if CmdSign not in list(CmdTargetDict.keys()): - CmdTargetDict[CmdSign] = Item.replace(Temp, CmdSign) + cmd = Item.replace(Temp, CmdSign) + if SingleCommandList[-1] in cmd: + CmdTargetDict[CmdSign] = + [cmd.replace(SingleCommandList[-1], "").rstrip(), + SingleCommandList[-1]] else: - CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1]) + # CmdTargetDict[CmdSign] = "%s %s" % + (CmdTargetDict[CmdSign], SingleCommandList[-1]) + + CmdTargetDict[CmdSign].append(SingleCommandList[-1]) Index = CommandList.index(Item) CommandList.pop(Index) if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign[3:].rsplit(TAB_SLASH, 1)[0]])): Cpplist = CmdCppDict[T.Target.SubDir] Cpplist.insert(0, '$(OBJLIST_%d): ' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir)) - T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign]) + source_files = CmdTargetDict[CmdSign][1:] + source_files.insert(0, " ") + if len(source_files)>2: + SaveFileOnChange(SaveFilePath, " ".join(source_files), False) + T.Commands[Index] = '%s\n\t%s $(cc_resp_%s)' % ( + ' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign][0], resp_file_number) + ToolsDef.append("cc_resp_%s = @%s" % + (resp_file_number, SaveFilePath)) + + elif len(source_files)<=2 and len(" ".join(CmdTargetDict[CmdSign][:2]))>GlobalData.gCommandMaxLength: + SaveFileOnChange(SaveFilePath, " ".join(source_files), False) + T.Commands[Index] = '%s\n\t%s $(cc_resp_%s)' % ( + ' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign][0], resp_file_number) + ToolsDef.append("cc_resp_%s = @%s" % + (resp_file_number, SaveFilePath)) + + else: + T.Commands[Index] = '%s\n\t%s' % (' + \\\n\t'.join(Cpplist), " ".join(CmdTargetDict[CmdSign])) else: T.Commands.pop(Index) return T, CmdSumDict, CmdTargetDict, CmdCppDict diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py index c3e6333217c4..b06ef42573fd 100644 --- a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py @@ -201,7 +201,17 @@ ${END} cc_options = line[len(cc_cmd)+2:].split() else: cc_options = line[len(cc_cmd):].split() - SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)} + for item in cc_options: + if not item.startswith("/"): + if item.endswith(".txt") and item.startswith("@"): + with open(item[1:], "r") as file: + source_files = file.readlines()[0].split() + SourceFileAbsPathMap = {os.path.basename(file): file for file in source_files if + os.path.exists(file)} + else: + if os.path.exists(item): + SourceFileAbsPathMap.update({os.path.basename(item): item.strip()}) + # SourceFileAbsPathMap = + {os.path.basename(item):item for item in cc_options if not + item.startswith("/") and os.path.exists(item)} if line in SourceFileAbsPathMap: current_source = line if current_source not in ModuleDepDict: -- 2.28.0.windows.1 ^ permalink raw reply related [flat|nested] 6+ messages in thread
end of thread, other threads:[~2020-11-10 5:44 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2020-11-05 1:26 [PATCH.V2] BaseTools: Limit command line length mingyuex.liang
2020-11-05 2:43 ` Bob Feng
2020-11-05 6:33 ` Mingyue Liang
2020-11-09 6:20 ` Bob Feng
[not found] ` <1645C2A4018343CE.24110@groups.io>
2020-11-10 4:10 ` [edk2-devel] " Bob Feng
2020-11-10 5:44 ` Mingyue Liang
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox