* [PATCH.V3] BaseTools: Limit command line length.
@ 2020-11-10 5:39 Mingyue Liang
2020-11-10 5:41 ` Bob Feng
0 siblings, 1 reply; 2+ messages in thread
From: Mingyue Liang @ 2020-11-10 5:39 UTC (permalink / raw)
To: devel; +Cc: mliang2x, Bob Feng, Liming Gao, Yuwei Chen
From: mliang2x <mingyuex.liang@intel.com>
REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2528
Currently, CL command contains multiple C files will be compiled,
and that caused command line too long, which may trigger build error.
In order to solve this issue, the following rules is used in this scene:
If the number of C files is greater than one, a txt file will be used
to record these C files, and replaces the corresponding content in
command line with the file name.
Else (only one C file listed in the command line), the length of the
whole CL command line will determine whether use a file to record. If
the length exceeds the limited max length, use the recording file; else
C file name directly listed in the command line
Signed-off-by: Mingyue Liang <mingyuex.liang@intel.com>
Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <gaoliming@byosoft.com.cn>
Cc: Yuwei Chen <yuwei.chen@intel.com>
---
V3:
Modify version update content description location.
BaseTools/Source/Python/AutoGen/GenMake.py | 41 +++++++++++++++----
.../Source/Python/AutoGen/IncludesAutoGen.py | 12 +++++-
2 files changed, 44 insertions(+), 9 deletions(-)
diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py
index b04d3f54368e..1cfac1cd82ca 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -576,7 +576,7 @@ cleanlib:
EdkLogger.error("build", AUTOGEN_ERROR, "Nothing to build",
ExtraData="[%s]" % str(MyAgo))
- self.ProcessBuildTargetList()
+ self.ProcessBuildTargetList(MyAgo.OutputDir, ToolsDef)
self.ParserGenerateFfsCmd()
# Generate macros used to represent input files
@@ -903,7 +903,7 @@ cleanlib:
BuildTargets[Target].Commands[i] = SingleCommand.replace('$(INC)', '').replace(FlagDict[Flag]['Macro'], RespMacro)
return RespDict
- def ProcessBuildTargetList(self):
+ def ProcessBuildTargetList(self, RespFile, ToolsDef):
#
# Search dependency file list for each source file
#
@@ -1004,6 +1004,7 @@ cleanlib:
self.ObjTargetDict[T.Target.SubDir] = set()
self.ObjTargetDict[T.Target.SubDir].add(NewFile)
for Type in self._AutoGenObject.Targets:
+ resp_file_number = 0
for T in self._AutoGenObject.Targets[Type]:
# Generate related macros if needed
if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros:
@@ -1045,7 +1046,10 @@ cleanlib:
Deps.append("$(%s)" % T.ListFileMacro)
if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE:
- T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict)
+ T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict,
+ CmdCppDict, DependencyDict, RespFile,
+ ToolsDef, resp_file_number)
+ resp_file_number += 1
TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps}
CmdLine = self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIST', '$(OBJLIST')
if T.Commands:
@@ -1062,7 +1066,9 @@ cleanlib:
AnnexeTargetDict = {"target": self.PlaceMacro(i.Path, self.Macros), "cmd": "", "deps": self.PlaceMacro(T.Target.Path, self.Macros)}
self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(AnnexeTargetDict))
- def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict):
+ def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict, RespFile, ToolsDef,
+ resp_file_number):
+ SaveFilePath = os.path.join(RespFile, "cc_resp_%s.txt" % resp_file_number)
if not CmdSumDict:
for item in self._AutoGenObject.Targets[Type]:
CmdSumDict[item.Target.SubDir] = item.Target.BaseName
@@ -1089,17 +1095,36 @@ cleanlib:
if Temp.startswith('/Fo'):
CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH)
break
- else: continue
+ else:
+ continue
if CmdSign not in list(CmdTargetDict.keys()):
- CmdTargetDict[CmdSign] = Item.replace(Temp, CmdSign)
+ cmd = Item.replace(Temp, CmdSign)
+ if SingleCommandList[-1] in cmd:
+ CmdTargetDict[CmdSign] = [cmd.replace(SingleCommandList[-1], "").rstrip(), SingleCommandList[-1]]
else:
- CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1])
+ # CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1])
+ CmdTargetDict[CmdSign].append(SingleCommandList[-1])
Index = CommandList.index(Item)
CommandList.pop(Index)
if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign[3:].rsplit(TAB_SLASH, 1)[0]])):
Cpplist = CmdCppDict[T.Target.SubDir]
Cpplist.insert(0, '$(OBJLIST_%d): ' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir))
- T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign])
+ source_files = CmdTargetDict[CmdSign][1:]
+ source_files.insert(0, " ")
+ if len(source_files)>2:
+ SaveFileOnChange(SaveFilePath, " ".join(source_files), False)
+ T.Commands[Index] = '%s\n\t%s $(cc_resp_%s)' % (
+ ' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign][0], resp_file_number)
+ ToolsDef.append("cc_resp_%s = @%s" % (resp_file_number, SaveFilePath))
+
+ elif len(source_files)<=2 and len(" ".join(CmdTargetDict[CmdSign][:2]))>GlobalData.gCommandMaxLength:
+ SaveFileOnChange(SaveFilePath, " ".join(source_files), False)
+ T.Commands[Index] = '%s\n\t%s $(cc_resp_%s)' % (
+ ' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign][0], resp_file_number)
+ ToolsDef.append("cc_resp_%s = @%s" % (resp_file_number, SaveFilePath))
+
+ else:
+ T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), " ".join(CmdTargetDict[CmdSign]))
else:
T.Commands.pop(Index)
return T, CmdSumDict, CmdTargetDict, CmdCppDict
diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
index c3e6333217c4..b06ef42573fd 100644
--- a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
@@ -201,7 +201,17 @@ ${END}
cc_options = line[len(cc_cmd)+2:].split()
else:
cc_options = line[len(cc_cmd):].split()
- SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)}
+ for item in cc_options:
+ if not item.startswith("/"):
+ if item.endswith(".txt") and item.startswith("@"):
+ with open(item[1:], "r") as file:
+ source_files = file.readlines()[0].split()
+ SourceFileAbsPathMap = {os.path.basename(file): file for file in source_files if
+ os.path.exists(file)}
+ else:
+ if os.path.exists(item):
+ SourceFileAbsPathMap.update({os.path.basename(item): item.strip()})
+ # SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)}
if line in SourceFileAbsPathMap:
current_source = line
if current_source not in ModuleDepDict:
--
2.28.0.windows.1
^ permalink raw reply related [flat|nested] 2+ messages in thread
* Re: [PATCH.V3] BaseTools: Limit command line length.
2020-11-10 5:39 [PATCH.V3] BaseTools: Limit command line length Mingyue Liang
@ 2020-11-10 5:41 ` Bob Feng
0 siblings, 0 replies; 2+ messages in thread
From: Bob Feng @ 2020-11-10 5:41 UTC (permalink / raw)
To: Liang, MingyueX, devel@edk2.groups.io
Cc: Liang, MingyueX, Liming Gao, Chen, Christine
Reviewed-by: Bob Feng <bob.c.feng@intel.com>
-----Original Message-----
From: Mingyue Liang <mingyuex.liang@intel.com>
Sent: Tuesday, November 10, 2020 1:40 PM
To: devel@edk2.groups.io
Cc: Liang, MingyueX <mingyuex.liang@intel.com>; Feng, Bob C <bob.c.feng@intel.com>; Liming Gao <gaoliming@byosoft.com.cn>; Chen, Christine <yuwei.chen@intel.com>
Subject: [PATCH.V3] BaseTools: Limit command line length.
From: mliang2x <mingyuex.liang@intel.com>
REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2528
Currently, CL command contains multiple C files will be compiled, and that caused command line too long, which may trigger build error.
In order to solve this issue, the following rules is used in this scene:
If the number of C files is greater than one, a txt file will be used to record these C files, and replaces the corresponding content in command line with the file name.
Else (only one C file listed in the command line), the length of the whole CL command line will determine whether use a file to record. If the length exceeds the limited max length, use the recording file; else C file name directly listed in the command line
Signed-off-by: Mingyue Liang <mingyuex.liang@intel.com>
Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <gaoliming@byosoft.com.cn>
Cc: Yuwei Chen <yuwei.chen@intel.com>
---
V3:
Modify version update content description location.
BaseTools/Source/Python/AutoGen/GenMake.py | 41 +++++++++++++++----
.../Source/Python/AutoGen/IncludesAutoGen.py | 12 +++++-
2 files changed, 44 insertions(+), 9 deletions(-)
diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py
index b04d3f54368e..1cfac1cd82ca 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -576,7 +576,7 @@ cleanlib:
EdkLogger.error("build", AUTOGEN_ERROR, "Nothing to build",
ExtraData="[%s]" % str(MyAgo))
- self.ProcessBuildTargetList()
+ self.ProcessBuildTargetList(MyAgo.OutputDir, ToolsDef)
self.ParserGenerateFfsCmd()
# Generate macros used to represent input files @@ -903,7 +903,7 @@ cleanlib:
BuildTargets[Target].Commands[i] = SingleCommand.replace('$(INC)', '').replace(FlagDict[Flag]['Macro'], RespMacro)
return RespDict
- def ProcessBuildTargetList(self):
+ def ProcessBuildTargetList(self, RespFile, ToolsDef):
#
# Search dependency file list for each source file
#
@@ -1004,6 +1004,7 @@ cleanlib:
self.ObjTargetDict[T.Target.SubDir] = set()
self.ObjTargetDict[T.Target.SubDir].add(NewFile)
for Type in self._AutoGenObject.Targets:
+ resp_file_number = 0
for T in self._AutoGenObject.Targets[Type]:
# Generate related macros if needed
if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros:
@@ -1045,7 +1046,10 @@ cleanlib:
Deps.append("$(%s)" % T.ListFileMacro)
if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Type == TAB_C_CODE_FILE:
- T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict)
+ T, CmdTarget, CmdTargetDict, CmdCppDict = self.ParserCCodeFile(T, Type, CmdSumDict, CmdTargetDict,
+ CmdCppDict, DependencyDict, RespFile,
+ ToolsDef, resp_file_number)
+ resp_file_number += 1
TargetDict = {"target": self.PlaceMacro(T.Target.Path, self.Macros), "cmd": "\n\t".join(T.Commands),"deps": CCodeDeps}
CmdLine = self._BUILD_TARGET_TEMPLATE.Replace(TargetDict).rstrip().replace('\t$(OBJLIST', '$(OBJLIST')
if T.Commands:
@@ -1062,7 +1066,9 @@ cleanlib:
AnnexeTargetDict = {"target": self.PlaceMacro(i.Path, self.Macros), "cmd": "", "deps": self.PlaceMacro(T.Target.Path, self.Macros)}
self.BuildTargetList.append(self._BUILD_TARGET_TEMPLATE.Replace(AnnexeTargetDict))
- def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict):
+ def ParserCCodeFile(self, T, Type, CmdSumDict, CmdTargetDict, CmdCppDict, DependencyDict, RespFile, ToolsDef,
+ resp_file_number):
+ SaveFilePath = os.path.join(RespFile, "cc_resp_%s.txt" %
+ resp_file_number)
if not CmdSumDict:
for item in self._AutoGenObject.Targets[Type]:
CmdSumDict[item.Target.SubDir] = item.Target.BaseName @@ -1089,17 +1095,36 @@ cleanlib:
if Temp.startswith('/Fo'):
CmdSign = '%s%s' % (Temp.rsplit(TAB_SLASH, 1)[0], TAB_SLASH)
break
- else: continue
+ else:
+ continue
if CmdSign not in list(CmdTargetDict.keys()):
- CmdTargetDict[CmdSign] = Item.replace(Temp, CmdSign)
+ cmd = Item.replace(Temp, CmdSign)
+ if SingleCommandList[-1] in cmd:
+ CmdTargetDict[CmdSign] =
+ [cmd.replace(SingleCommandList[-1], "").rstrip(),
+ SingleCommandList[-1]]
else:
- CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1])
+ # CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1])
+
+ CmdTargetDict[CmdSign].append(SingleCommandList[-1])
Index = CommandList.index(Item)
CommandList.pop(Index)
if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign[3:].rsplit(TAB_SLASH, 1)[0]])):
Cpplist = CmdCppDict[T.Target.SubDir]
Cpplist.insert(0, '$(OBJLIST_%d): ' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir))
- T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign])
+ source_files = CmdTargetDict[CmdSign][1:]
+ source_files.insert(0, " ")
+ if len(source_files)>2:
+ SaveFileOnChange(SaveFilePath, " ".join(source_files), False)
+ T.Commands[Index] = '%s\n\t%s $(cc_resp_%s)' % (
+ ' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign][0], resp_file_number)
+ ToolsDef.append("cc_resp_%s = @%s" %
+ (resp_file_number, SaveFilePath))
+
+ elif len(source_files)<=2 and len(" ".join(CmdTargetDict[CmdSign][:2]))>GlobalData.gCommandMaxLength:
+ SaveFileOnChange(SaveFilePath, " ".join(source_files), False)
+ T.Commands[Index] = '%s\n\t%s $(cc_resp_%s)' % (
+ ' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign][0], resp_file_number)
+ ToolsDef.append("cc_resp_%s = @%s" %
+ (resp_file_number, SaveFilePath))
+
+ else:
+ T.Commands[Index] = '%s\n\t%s' % ('
+ \\\n\t'.join(Cpplist), " ".join(CmdTargetDict[CmdSign]))
else:
T.Commands.pop(Index)
return T, CmdSumDict, CmdTargetDict, CmdCppDict diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
index c3e6333217c4..b06ef42573fd 100644
--- a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
@@ -201,7 +201,17 @@ ${END}
cc_options = line[len(cc_cmd)+2:].split()
else:
cc_options = line[len(cc_cmd):].split()
- SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)}
+ for item in cc_options:
+ if not item.startswith("/"):
+ if item.endswith(".txt") and item.startswith("@"):
+ with open(item[1:], "r") as file:
+ source_files = file.readlines()[0].split()
+ SourceFileAbsPathMap = {os.path.basename(file): file for file in source_files if
+ os.path.exists(file)}
+ else:
+ if os.path.exists(item):
+ SourceFileAbsPathMap.update({os.path.basename(item): item.strip()})
+ # SourceFileAbsPathMap =
+ {os.path.basename(item):item for item in cc_options if not
+ item.startswith("/") and os.path.exists(item)}
if line in SourceFileAbsPathMap:
current_source = line
if current_source not in ModuleDepDict:
--
2.28.0.windows.1
^ permalink raw reply related [flat|nested] 2+ messages in thread
end of thread, other threads:[~2020-11-10 5:41 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2020-11-10 5:39 [PATCH.V3] BaseTools: Limit command line length Mingyue Liang
2020-11-10 5:41 ` Bob Feng
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox